From 8815da05fc8b8eace2a998fe64bcb7bd329275fb Mon Sep 17 00:00:00 2001 From: wangyaming Date: Wed, 31 May 2017 11:30:24 +0800 Subject: [PATCH 1/3] add pushgateway related functions --- config/v2/configV2.go | 53 +++++++- exporter/grok.go | 12 ++ exporter/metrics.go | 258 +++++++++++++++++++++++++++++++-------- exporter/metrics_test.go | 10 +- grok_exporter.go | 134 +++++++++++++++++++- 5 files changed, 396 insertions(+), 71 deletions(-) diff --git a/config/v2/configV2.go b/config/v2/configV2.go index 0804081d..5466ac9f 100644 --- a/config/v2/configV2.go +++ b/config/v2/configV2.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/fstab/grok_exporter/templates" "gopkg.in/yaml.v2" + "regexp" ) func Unmarshal(config []byte) (*Config, error) { @@ -42,7 +43,8 @@ func (cfg *Config) String() string { } type GlobalConfig struct { - ConfigVersion int `yaml:"config_version,omitempty"` + ConfigVersion int `yaml:"config_version,omitempty"` + PushgatewayAddr string `yaml:"pushgateway_addr,omitempty"` // add pushgateway address : } type InputConfig struct { @@ -57,10 +59,19 @@ type GrokConfig struct { } type MetricConfig struct { - Type string `yaml:",omitempty"` - Name string `yaml:",omitempty"` - Help string `yaml:",omitempty"` - Match string `yaml:",omitempty"` + Type string `yaml:",omitempty"` + Name string `yaml:",omitempty"` + Help string `yaml:",omitempty"` + Match string `yaml:",omitempty"` + + /**************pushgateway related configs*******************/ + Pushgateway bool `yaml:",omitempty"` //flag for if push metric to pushgateway or not + JobName string `yaml:"job_name,omitempty"` //job name used to push metric to pushgateway + DeleteMatch string `yaml:"delete_match,omitempty"` //if match, metric will be deleted from pushgateway + GroupingKey map[string]string `yaml:"grouping_key,omitempty"` //grouping key used to push and delete metric from pushgateway + GroupTemplates []templates.Template `yaml:"-"` + /**************end of pushgateway related configs************/ + Value string `yaml:",omitempty"` Cumulative bool `yaml:",omitempty"` Buckets []float64 `yaml:",flow,omitempty"` @@ -138,7 +149,11 @@ func (c *ServerConfig) addDefaults() { } func (cfg *Config) validate() error { - err := cfg.Input.validate() + err := cfg.Global.validate() + if err != nil { + return err + } + err = cfg.Input.validate() if err != nil { return err } @@ -157,6 +172,17 @@ func (cfg *Config) validate() error { return nil } +func (c *GlobalConfig) validate() error { + //ignore version validation + if len(c.PushgatewayAddr) > 0 { + reg := regexp.MustCompile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:[0-9]{1,5}") + result := reg.FindString(c.PushgatewayAddr) + if result == "" { + return fmt.Errorf("Not valid pushgateway address, usage: >:.") + } + } + return nil +} func (c *InputConfig) validate() error { switch { case c.Type == "stdin": @@ -236,6 +262,11 @@ func (c *MetricConfig) validate() error { return fmt.Errorf("Invalid metric configuration: 'metrics.buckets' cannot be used for %v metrics.", c.Type) } // Labels and value are validated in InitTemplates() + //validate pushgateway related configs + if len(c.DeleteMatch) > 0 && !c.Pushgateway { + return fmt.Errorf("Invalid metric configuration: 'delete_match' can only be defined when 'pushgateway = true'.") + } + //grouping key is validated in InitTemplates() return nil } @@ -289,6 +320,16 @@ func (metric *MetricConfig) InitTemplates() error { } metric.LabelTemplates = append(metric.LabelTemplates, tmplt) } + // validate grouping key + metric.GroupTemplates = make([]templates.Template, 0, len(metric.GroupingKey)) + for name, templateString := range metric.GroupingKey { + tmplt, err = templates.New(name, templateString) + if err != nil { + return fmt.Errorf(msg, fmt.Sprintf("groupingKey %v", metric.Name), name, err.Error()) + } + metric.GroupTemplates = append(metric.GroupTemplates, tmplt) + } + if len(metric.Value) > 0 { metric.ValueTemplate, err = templates.New("__value__", metric.Value) if err != nil { diff --git a/exporter/grok.go b/exporter/grok.go index a2e3fc48..65121549 100644 --- a/exporter/grok.go +++ b/exporter/grok.go @@ -52,6 +52,18 @@ func VerifyFieldNames(m *v2.MetricConfig, regex *OnigurumaRegexp) error { return nil } +func VerifyGroupingKeyField(m *v2.MetricConfig, regex *OnigurumaRegexp) error { + for _, template := range m.GroupTemplates { + for _, grokFieldName := range template.ReferencedGrokFields() { + if !regex.HasCaptureGroup(grokFieldName) { + return fmt.Errorf("%v: error in label %v: grok field %v not found in match pattern", m.Name, template.Name(), grokFieldName) + } + } + } + + return nil +} + // PATTERN_RE matches the %{..} patterns. There are three possibilities: // 1) %{USER} - grok pattern // 2) %{IP:clientip} - grok pattern with name diff --git a/exporter/metrics.go b/exporter/metrics.go index edb6e1d8..409d43a9 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -25,9 +25,12 @@ import ( type Metric interface { Name() string Collector() prometheus.Collector + MetricVec() *prometheus.MetricVec // Returns true if the line matched, and false if the line didn't match. - Process(line string) (bool, error) + Process(line string) (bool, bool, map[string]string, []string, error) + NeedPush() bool + JobName() string } // Represents a Prometheus Counter @@ -36,20 +39,34 @@ type incMetric struct { regex *OnigurumaRegexp labels []templates.Template collector prometheus.Collector - incFunc func(m *OnigurumaMatchResult) error + metricVec *prometheus.MetricVec + //pushgateway related configs + delete_regex *OnigurumaRegexp + pushgateway bool + job_name string + groupingKey []templates.Template + + incFunc func(m *OnigurumaMatchResult) error } // Represents a Prometheus Gauge, Histogram, or Summary type observeMetric struct { - name string - regex *OnigurumaRegexp - value templates.Template - labels []templates.Template + name string + regex *OnigurumaRegexp + value templates.Template + labels []templates.Template + //pushgateway related configs + delete_regex *OnigurumaRegexp + pushgateway bool + job_name string + groupingKey []templates.Template + collector prometheus.Collector + metricVec *prometheus.MetricVec observeFunc func(m *OnigurumaMatchResult, val float64) error } -func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { counterOpts := prometheus.CounterOpts{ Name: cfg.Name, Help: cfg.Help, @@ -57,9 +74,13 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular counter counter := prometheus.NewCounter(counterOpts) return &incMetric{ - name: cfg.Name, - regex: regex, - collector: counter, + name: cfg.Name, + regex: regex, + collector: counter, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, incFunc: func(_ *OnigurumaMatchResult) error { counter.Inc() return nil @@ -68,10 +89,15 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // counterVec counterVec := prometheus.NewCounterVec(counterOpts, prometheusLabels(cfg.LabelTemplates)) result := &incMetric{ - name: cfg.Name, - regex: regex, - labels: cfg.LabelTemplates, - collector: counterVec, + name: cfg.Name, + regex: regex, + labels: cfg.LabelTemplates, + collector: counterVec, + metricVec: counterVec.MetricVec, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, incFunc: func(m *OnigurumaMatchResult) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -84,7 +110,7 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } } -func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { gaugeOpts := prometheus.GaugeOpts{ Name: cfg.Name, Help: cfg.Help, @@ -92,10 +118,14 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular gauge gauge := prometheus.NewGauge(gaugeOpts) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: gauge, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: gauge, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { if cfg.Cumulative { gauge.Add(val) @@ -108,11 +138,16 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // gaugeVec gaugeVec := prometheus.NewGaugeVec(gaugeOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: gaugeVec, - labels: cfg.LabelTemplates, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: gaugeVec, + metricVec: gaugeVec.MetricVec, + labels: cfg.LabelTemplates, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -128,7 +163,7 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } } -func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { histogramOpts := prometheus.HistogramOpts{ Name: cfg.Name, Help: cfg.Help, @@ -139,10 +174,14 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular histogram histogram := prometheus.NewHistogram(histogramOpts) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: histogram, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: histogram, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { histogram.Observe(val) return nil @@ -151,11 +190,16 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // histogramVec histogramVec := prometheus.NewHistogramVec(histogramOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: histogramVec, - labels: cfg.LabelTemplates, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: histogramVec, + metricVec: histogramVec.MetricVec, + labels: cfg.LabelTemplates, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -167,7 +211,7 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } } -func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { summaryOpts := prometheus.SummaryOpts{ Name: cfg.Name, Help: cfg.Help, @@ -178,10 +222,14 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular summary summary := prometheus.NewSummary(summaryOpts) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: summary, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: summary, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { summary.Observe(val) return nil @@ -190,11 +238,16 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // summaryVec summaryVec := prometheus.NewSummaryVec(summaryOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: summaryVec, - labels: cfg.LabelTemplates, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: summaryVec, + metricVec: summaryVec.MetricVec, + labels: cfg.LabelTemplates, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -207,40 +260,91 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } // Return: true if the line matched, false if it didn't match. -func (m *incMetric) Process(line string) (bool, error) { +func (m *incMetric) Process(line string) (bool, bool, map[string]string, []string, error) { matchResult, err := m.regex.Match(line) if err != nil { - return false, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + } + var deleteMatch *OnigurumaMatchResult = nil + var e error + if m.delete_regex != nil { + deleteMatch, e = m.delete_regex.Match(line) + if e != nil { + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, e.Error()) + } } + defer matchResult.Free() + if deleteMatch != nil { + defer deleteMatch.Free() + } + + //metric can either be pushed or deleted, CANNOT be both in single line processing if matchResult.IsMatch() { err = m.incFunc(matchResult) - return true, err + metricLabelValues, err := labelValues(matchResult, m.labels) + groupingKey, e := evalGroupingKey(matchResult, m.groupingKey) + if e != nil { + return true, false, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, e.Error()) + } + return true, false, groupingKey, metricLabelValues, err + } else { - return false, nil + if deleteMatch != nil && deleteMatch.IsMatch() { + groupingKey, e := evalGroupingKey(deleteMatch, m.groupingKey) + if e != nil { + return false, true, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, e.Error()) + } + return false, true, groupingKey, nil, nil + } + return false, false, nil, nil, nil } } // Return: true if the line matched, false if it didn't match. -func (m *observeMetric) Process(line string) (bool, error) { +func (m *observeMetric) Process(line string) (bool, bool, map[string]string, []string, error) { matchResult, err := m.regex.Match(line) if err != nil { - return false, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + } + var deleteMatch *OnigurumaMatchResult = nil + var e error + if m.delete_regex != nil { + deleteMatch, e = m.delete_regex.Match(line) + if e != nil { + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, e.Error()) + } } defer matchResult.Free() + if deleteMatch != nil { + defer deleteMatch.Free() + } + if matchResult.IsMatch() { stringVal, err := evalTemplate(matchResult, m.value) if err != nil { - return true, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + return true, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) } floatVal, err := strconv.ParseFloat(stringVal, 64) if err != nil { - return true, fmt.Errorf("error while processing metric %v: value '%v' matches '%v', which is not a valid number.", m.name, m.value, stringVal) + return true, false, nil, nil, fmt.Errorf("error while processing metric %v: value '%v' matches '%v', which is not a valid number.", m.name, m.value, stringVal) } err = m.observeFunc(matchResult, floatVal) - return true, err + metricLabelValues, err := labelValues(matchResult, m.labels) + groupingKey, e := evalGroupingKey(matchResult, m.groupingKey) + if e != nil { + return true, false, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, e.Error()) + } + return true, false, groupingKey, metricLabelValues, err } else { - return false, nil + if deleteMatch != nil && deleteMatch.IsMatch() { + groupingKey, err := evalGroupingKey(deleteMatch, m.groupingKey) + if err != nil { + return false, true, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, err.Error()) + } + return false, true, groupingKey, nil, nil + } + return false, false, nil, nil, nil } } @@ -260,6 +364,52 @@ func (m *observeMetric) Collector() prometheus.Collector { return m.collector } +func (m *incMetric) MetricVec() *prometheus.MetricVec { + return m.metricVec +} + +func (m *observeMetric) MetricVec() *prometheus.MetricVec { + return m.metricVec +} + +func (m *incMetric) NeedPush() bool { + return m.pushgateway +} + +func (m *observeMetric) NeedPush() bool { + return m.pushgateway +} + +func (m *incMetric) JobName() string { + if len(m.job_name) != 0 { + return m.job_name + } else { + return "grok_exporter" + } +} + +func (m *observeMetric) JobName() string { + if len(m.job_name) != 0 { + return m.job_name + } else { + return "grok_exporter" + } +} + +func evalGroupingKey(matchResult *OnigurumaMatchResult, templates []templates.Template) (map[string]string, error) { + result := make(map[string]string, len(templates)) + for _, t := range templates { + value, err := evalTemplate(matchResult, t) + if err != nil { + return nil, err + } + result[t.Name()] = value + + } + //fmt.Println("[DEBUG] got groupingKey: %s", result) + return result, nil +} + func labelValues(matchResult *OnigurumaMatchResult, templates []templates.Template) ([]string, error) { result := make([]string, 0, len(templates)) for _, t := range templates { diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index c54deab8..83d313b1 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -30,7 +30,7 @@ func TestCounterVec(t *testing.T) { "error_message": "{{.message}}", }, }) - counter := NewCounterMetric(counterCfg, regex) + counter := NewCounterMetric(counterCfg, regex, nil) counter.Process("some unrelated line") counter.Process("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F= rejected RCPT : relay not permitted") counter.Process("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F= rejected RCPT : Unrouteable address") @@ -57,7 +57,7 @@ func TestCounter(t *testing.T) { counterCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "exim_rejected_rcpt_total", }) - counter := NewCounterMetric(counterCfg, regex) + counter := NewCounterMetric(counterCfg, regex, nil) counter.Process("some unrelated line") counter.Process("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F= rejected RCPT : relay not permitted") @@ -99,7 +99,7 @@ func TestGauge(t *testing.T) { Name: "temperature", Value: "{{.temperature}}", }) - gauge := NewGaugeMetric(gaugeCfg, regex) + gauge := NewGaugeMetric(gaugeCfg, regex, nil) gauge.Process("Temperature in Berlin: 32") gauge.Process("Temperature in Moscow: -5") @@ -123,7 +123,7 @@ func TestGaugeCumulative(t *testing.T) { Value: "{{.temperature}}", Cumulative: true, }) - gauge := NewGaugeMetric(gaugeCfg, regex) + gauge := NewGaugeMetric(gaugeCfg, regex, nil) gauge.Process("Temperature in Berlin: 32") gauge.Process("Temperature in Moscow: -5") @@ -149,7 +149,7 @@ func TestGaugeVec(t *testing.T) { "city": "{{.city}}", }, }) - gauge := NewGaugeMetric(gaugeCfg, regex) + gauge := NewGaugeMetric(gaugeCfg, regex, nil) gauge.Process("Temperature in Berlin: 32") gauge.Process("Temperature in Moscow: -5") diff --git a/grok_exporter.go b/grok_exporter.go index f32799d5..33153a14 100644 --- a/grok_exporter.go +++ b/grok_exporter.go @@ -15,6 +15,7 @@ package main import ( + "bytes" "flag" "fmt" "github.com/fstab/grok_exporter/config" @@ -22,8 +23,13 @@ import ( "github.com/fstab/grok_exporter/exporter" "github.com/fstab/grok_exporter/tailer" "github.com/prometheus/client_golang/prometheus" + //"github.com/prometheus/client_golang/prometheus/push" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" "net/http" + "net/url" "os" + "strings" "time" ) @@ -68,7 +74,7 @@ func main() { tail, err := startTailer(cfg) exitOnError(err) - fmt.Print(startMsg(cfg)) + fmt.Println(startMsg(cfg)) serverErrors := startServer(cfg, "/metrics", prometheus.Handler()) for { @@ -81,17 +87,35 @@ func main() { matched := false for _, metric := range metrics { start := time.Now() - match, err := metric.Process(line) + match, delete_match, groupingKey, labelValues, err := metric.Process(line) + //fmt.Println(fmt.Sprintf("[DEBUG] Process result: match: %s, delete_match: %s, groupingKey: %s, err: %s", match, delete_match, groupingKey, err)) + + pushFlag := true if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Skipping log line: %v\n", err.Error()) fmt.Fprintf(os.Stderr, "%v\n", line) nErrorsByMetric.WithLabelValues(metric.Name()).Inc() + pushFlag = false } if match { + if metric.NeedPush() && pushFlag { + err := pushMetric(metric, cfg.Global.PushgatewayAddr, groupingKey, labelValues) + if err != nil { + //fmt.Println(fmt.Sprintf("[DEBUG] Push error: %s", err)) + fmt.Errorf("Error pushing metric %v to pushgateway.", metric.Name()) + } + } + nMatchesByMetric.WithLabelValues(metric.Name()).Inc() procTimeMicrosecondsByMetric.WithLabelValues(metric.Name()).Add(float64(time.Since(start).Nanoseconds() / int64(1000))) matched = true } + if delete_match { + err := deleteMetric(metric, cfg.Global.PushgatewayAddr, groupingKey) + if err != nil { + fmt.Errorf("Error deleting metric %v from pushgateway.", metric.Name()) + } + } } if matched { nLinesTotal.WithLabelValues(number_of_lines_matched_label).Inc() @@ -102,6 +126,89 @@ func main() { } } +func pushMetric(m exporter.Metric, pushUrl string, groupingKey map[string]string, labelValues []string) error { + //fmt.Println(fmt.Sprintf("[DEBUG] Pushing metric %s with labels %s to pushgateway %s of job %s", m.Name(), groupingKey, pushUrl, m.JobName())) + r := prometheus.NewRegistry() + if err := r.Register(m.Collector()); err != nil { + return err + } + err := doRequest(m.JobName(), groupingKey, pushUrl, r, "POST") + if err != nil { + return err + } + //remove metric from collector + if m.MetricVec() != nil { + m.MetricVec().DeleteLabelValues(labelValues...) + } + return nil +} + +func deleteMetric(m exporter.Metric, deleteUrl string, groupingKey map[string]string) error { + //fmt.Println(fmt.Sprintf("[DEBUG] Deleting metric %s with labels %s from pushgateway %s of job %s", m.Name(), groupingKey, deleteUrl, m.JobName())) + return doRequest(m.JobName(), groupingKey, deleteUrl, nil, "DELETE") + +} + +func doRequest(job string, groupingKey map[string]string, targetUrl string, g prometheus.Gatherer, method string) error { + if !strings.Contains(targetUrl, "://") { + targetUrl = "http://" + targetUrl + } + if strings.HasSuffix(targetUrl, "/") { + targetUrl = targetUrl[:len(targetUrl)-1] + } + + if strings.Contains(job, "/") { + return fmt.Errorf("job contains '/' : %s", job) + } + urlComponents := []string{url.QueryEscape(job)} + for ln, lv := range groupingKey { + if !model.LabelName(ln).IsValid() { + return fmt.Errorf("groupingKey label has invalid name: %s", ln) + } + if strings.Contains(lv, "/") { + return fmt.Errorf("value of groupingKey label %s contains '/': %s", ln, lv) + } + urlComponents = append(urlComponents, ln, lv) + } + + targetUrl = fmt.Sprintf("%s/metrics/job/%s", targetUrl, strings.Join(urlComponents, "/")) + + buf := &bytes.Buffer{} + enc := expfmt.NewEncoder(buf, expfmt.FmtProtoDelim) + if g != nil { + mfs, err := g.Gather() + if err != nil { + return err + } + for _, mf := range mfs { + //ignore checking for pre-existing labels + enc.Encode(mf) + } + } + + var request *http.Request + var err error + if method == "DELETE" { + request, err = http.NewRequest(method, targetUrl, nil) + } else { + request, err = http.NewRequest(method, targetUrl, buf) + } + + if err != nil { + return err + } + request.Header.Set("Content-Type", string(expfmt.FmtProtoDelim)) + response, err := http.DefaultClient.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode != 202 { + return fmt.Errorf("unexpected status code %d, method %s", response.StatusCode, method) + } + return nil +} + func startMsg(cfg *v2.Config) string { host := "localhost" if len(cfg.Server.Host) > 0 { @@ -161,18 +268,33 @@ func createMetrics(cfg *v2.Config, patterns *exporter.Patterns, libonig *exporte if err != nil { return nil, fmt.Errorf("failed to initialize metric %v: %v", m.Name, err.Error()) } + + var delete_regex *exporter.OnigurumaRegexp = nil + + if len(m.DeleteMatch) != 0 { + delete_regex, err = exporter.Compile(m.DeleteMatch, patterns, libonig) + if err != nil { + return nil, fmt.Errorf("failed to initialize metric %v: %v", m.Name, err.Error()) + } + err = exporter.VerifyGroupingKeyField(m, delete_regex) + if err != nil { + return nil, fmt.Errorf("failed to initialize metric %v: %v", m.Name, err.Error()) + } + } + switch m.Type { case "counter": - result = append(result, exporter.NewCounterMetric(m, regex)) + result = append(result, exporter.NewCounterMetric(m, regex, delete_regex)) case "gauge": - result = append(result, exporter.NewGaugeMetric(m, regex)) + result = append(result, exporter.NewGaugeMetric(m, regex, delete_regex)) case "histogram": - result = append(result, exporter.NewHistogramMetric(m, regex)) + result = append(result, exporter.NewHistogramMetric(m, regex, delete_regex)) case "summary": - result = append(result, exporter.NewSummaryMetric(m, regex)) + result = append(result, exporter.NewSummaryMetric(m, regex, delete_regex)) default: return nil, fmt.Errorf("Failed to initialize metrics: Metric type %v is not supported.", m.Type) } + } return result, nil } From a0a08ddb842f54dd3492338adb7bcf5fb9c2dfd9 Mon Sep 17 00:00:00 2001 From: wangyaming Date: Fri, 9 Feb 2018 17:11:34 +0800 Subject: [PATCH 2/3] change dependencies to github.com/yamingwa/* --- CONFIG.md | 2 +- CONFIG_v1.md | 2 +- README.md | 30 +++++++++++++++--------------- config/config.go | 4 ++-- config/v1/configV1.go | 2 +- config/v2/configV2.go | 2 +- exporter/bufferedTailer.go | 2 +- exporter/grok.go | 2 +- exporter/grok_test.go | 2 +- exporter/metrics.go | 4 ++-- exporter/metrics_test.go | 2 +- grok_exporter.go | 8 ++++---- integration-test.sh | 2 +- release.sh | 12 ++++++------ vendor/update.sh | 2 +- 15 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 173de77a..76126094 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -329,7 +329,7 @@ server: [gauge metric]: https://prometheus.io/docs/concepts/metric_types/#gauge [summary metric]: https://prometheus.io/docs/concepts/metric_types/#summary [histogram metric]: https://prometheus.io/docs/concepts/metric_types/#histogram -[release]: https://github.com/fstab/grok_exporter/releases +[release]: https://github.com/yamingwa/grok_exporter/releases [Prometheus metric types]: https://prometheus.io/docs/concepts/metric_types [Grok documentation]: https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html [histograms and summaries]: https://prometheus.io/docs/practices/histograms/ diff --git a/CONFIG_v1.md b/CONFIG_v1.md index 9bcf135e..99da1be2 100644 --- a/CONFIG_v1.md +++ b/CONFIG_v1.md @@ -296,7 +296,7 @@ server: [gauge metric]: https://prometheus.io/docs/concepts/metric_types/#gauge [summary metric]: https://prometheus.io/docs/concepts/metric_types/#summary [histogram metric]: https://prometheus.io/docs/concepts/metric_types/#histogram -[release]: https://github.com/fstab/grok_exporter/releases +[release]: https://github.com/yamingwa/grok_exporter/releases [Prometheus metric types]: https://prometheus.io/docs/concepts/metric_types [Prometheus data model documentation]: https://prometheus.io/docs/concepts/data_model [Grok documentation]: https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html diff --git a/README.md b/README.md index ecbde91e..fee01135 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/fstab/grok_exporter.svg?branch=master)](https://travis-ci.org/fstab/grok_exporter) [![Build status](https://ci.appveyor.com/api/projects/status/d8aq0pa3yfoapd69?svg=true)](https://ci.appveyor.com/project/fstab/grok-exporter) [![Coverage Status](https://coveralls.io/repos/github/fstab/grok_exporter/badge.svg?branch=master)](https://coveralls.io/github/fstab/grok_exporter?branch=master) +[![Build Status](https://travis-ci.org/yamingwa/grok_exporter.svg?branch=master)](https://travis-ci.org/yamingwa/grok_exporter) [![Build status](https://ci.appveyor.com/api/projects/status/d8aq0pa3yfoapd69?svg=true)](https://ci.appveyor.com/project/yamingwa/grok-exporter) [![Coverage Status](https://coveralls.io/repos/github/yamingwa/grok_exporter/badge.svg?branch=master)](https://coveralls.io/github/yamingwa/grok_exporter?branch=master) grok_exporter ============= @@ -61,9 +61,9 @@ Status Operating system support: -* Linux 64 Bit: [Supported](https://travis-ci.org/fstab/grok_exporter) -* Windows 64 Bit: [Supported](https://ci.appveyor.com/project/fstab/grok-exporter) -* mac OS 64 Bit: [Supported](https://travis-ci.org/fstab/grok_exporter) +* Linux 64 Bit: [Supported](https://travis-ci.org/yamingwa/grok_exporter) +* Windows 64 Bit: [Supported](https://ci.appveyor.com/project/yamingwa/grok-exporter) +* mac OS 64 Bit: [Supported](https://travis-ci.org/yamingwa/grok_exporter) Grok pattern support: @@ -86,7 +86,7 @@ In order to compile `grok_exporter` from source, you need [Go] installed and `$G The current version of `brew install oniguruma` will install Oniguruma 6.1.0. Because of [this bug](https://github.com/kkos/oniguruma/issues/23) version 6.1.0 will not work with grok_exporter. Use the following to install the stable 5.9.6 version: ```bash -brew install fstab/oniguruma/oniguruma-5.9.6 +brew install yamingwa/oniguruma/oniguruma-5.9.6 ``` **Installing the Oniguruma library on Ubuntu Linux** @@ -112,8 +112,8 @@ cd onig-5.9.6 && ./configure && make && make install With Oniguruma 5.9.6 installed, download and compile `grok_exporter` as follows: ```bash -go get github.com/fstab/grok_exporter -cd $GOPATH/src/github.com/fstab/grok_exporter +go get github.com/yamingwa/grok_exporter +cd $GOPATH/src/github.com/yamingwa/grok_exporter git submodule update --init --recursive ``` @@ -129,8 +129,8 @@ User documentation is included in the [GitHub repository]: Developer notes are available on the [GitHub Wiki pages]: -* [tailer (tail -f)](https://github.com/fstab/grok_exporter/wiki/tailer-(tail-%E2%80%90f)) -* [About the Regular Expression Library](https://github.com/fstab/grok_exporter/wiki/About-the-Regular-Expression-Library) +* [tailer (tail -f)](https://github.com/yamingwa/grok_exporter/wiki/tailer-(tail-%E2%80%90f)) +* [About the Regular Expression Library](https://github.com/yamingwa/grok_exporter/wiki/About-the-Regular-Expression-Library) External documentation: @@ -143,7 +143,7 @@ Contact * For feature requests, bugs reports, etc: Please open a GitHub issue. * For bug fixes, contributions, etc: Create a pull request. -* Questions? Contact me at fabian@fstab.de. +* Questions? Contact me at fabian@yamingwa.de. Related Projects ---------------- @@ -165,7 +165,7 @@ You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE- [Go]: https://golang.org/ [Oniguruma]: https://github.com/kkos/oniguruma [screenshot.png]: screenshot.png -[releases]: https://github.com/fstab/grok_exporter/releases +[releases]: https://github.com/yamingwa/grok_exporter/releases [http://localhost:9144/metrics]: http://localhost:9144/metrics [CONFIG.md]: CONFIG.md [BUILTIN.md]: BUILTIN.md @@ -177,11 +177,11 @@ You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE- [libpcre]: http://www.pcre.org [rubex]: https://github.com/moovweb/rubex [http://www.apache.org/licenses/LICENSE-2.0]: http://www.apache.org/licenses/LICENSE-2.0 -[more CPU efficient]: https://github.com/fstab/grok_exporter/wiki/About-the-Regular-Expression-Library +[more CPU efficient]: https://github.com/yamingwa/grok_exporter/wiki/About-the-Regular-Expression-Library [fsnotify]: https://github.com/fsnotify/fsnotify -[might be an obstacle]: https://github.com/fstab/grok_exporter/wiki/tailer-(tail-%E2%80%90f) -[GitHub Wiki pages]: https://github.com/fstab/grok_exporter/wiki -[GitHub repository]: https://github.com/fstab/grok_exporter +[might be an obstacle]: https://github.com/yamingwa/grok_exporter/wiki/tailer-(tail-%E2%80%90f) +[GitHub Wiki pages]: https://github.com/yamingwa/grok_exporter/wiki +[GitHub repository]: https://github.com/yamingwa/grok_exporter [Counter]: https://prometheus.io/docs/concepts/metric_types/#counter [Gauge]: https://prometheus.io/docs/concepts/metric_types/#gauge [Histogram]: https://prometheus.io/docs/concepts/metric_types/#histogram diff --git a/config/config.go b/config/config.go index 7fe890ab..4fcc431e 100644 --- a/config/config.go +++ b/config/config.go @@ -16,8 +16,8 @@ package config import ( "fmt" - "github.com/fstab/grok_exporter/config/v1" - "github.com/fstab/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/config/v1" + "github.com/yamingwa/grok_exporter/config/v2" "io/ioutil" "regexp" "strconv" diff --git a/config/v1/configV1.go b/config/v1/configV1.go index 1ca9aa51..61781fb5 100644 --- a/config/v1/configV1.go +++ b/config/v1/configV1.go @@ -16,7 +16,7 @@ package v1 import ( "fmt" - "github.com/fstab/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/config/v2" "gopkg.in/yaml.v2" ) diff --git a/config/v2/configV2.go b/config/v2/configV2.go index 5466ac9f..d51acad1 100644 --- a/config/v2/configV2.go +++ b/config/v2/configV2.go @@ -16,7 +16,7 @@ package v2 import ( "fmt" - "github.com/fstab/grok_exporter/templates" + "github.com/yamingwa/grok_exporter/templates" "gopkg.in/yaml.v2" "regexp" ) diff --git a/exporter/bufferedTailer.go b/exporter/bufferedTailer.go index fa69d161..52678c18 100644 --- a/exporter/bufferedTailer.go +++ b/exporter/bufferedTailer.go @@ -16,7 +16,7 @@ package exporter import ( "container/list" - "github.com/fstab/grok_exporter/tailer" + "github.com/yamingwa/grok_exporter/tailer" "github.com/prometheus/client_golang/prometheus" "log" "sync" diff --git a/exporter/grok.go b/exporter/grok.go index 65121549..14c8cc56 100644 --- a/exporter/grok.go +++ b/exporter/grok.go @@ -16,7 +16,7 @@ package exporter import ( "fmt" - "github.com/fstab/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/config/v2" "regexp" "strings" ) diff --git a/exporter/grok_test.go b/exporter/grok_test.go index c12d9b19..581fb622 100644 --- a/exporter/grok_test.go +++ b/exporter/grok_test.go @@ -15,7 +15,7 @@ package exporter import ( - "github.com/fstab/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/config/v2" "gopkg.in/yaml.v2" "strings" "testing" diff --git a/exporter/metrics.go b/exporter/metrics.go index 409d43a9..915cb567 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -16,8 +16,8 @@ package exporter import ( "fmt" - "github.com/fstab/grok_exporter/config/v2" - "github.com/fstab/grok_exporter/templates" + "github.com/yamingwa/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/templates" "github.com/prometheus/client_golang/prometheus" "strconv" ) diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index 83d313b1..eb555cfb 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -15,7 +15,7 @@ package exporter import ( - "github.com/fstab/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/config/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_model/go" "reflect" diff --git a/grok_exporter.go b/grok_exporter.go index 33153a14..2f3e348a 100644 --- a/grok_exporter.go +++ b/grok_exporter.go @@ -18,10 +18,10 @@ import ( "bytes" "flag" "fmt" - "github.com/fstab/grok_exporter/config" - "github.com/fstab/grok_exporter/config/v2" - "github.com/fstab/grok_exporter/exporter" - "github.com/fstab/grok_exporter/tailer" + "github.com/yamingwa/grok_exporter/config" + "github.com/yamingwa/grok_exporter/config/v2" + "github.com/yamingwa/grok_exporter/exporter" + "github.com/yamingwa/grok_exporter/tailer" "github.com/prometheus/client_golang/prometheus" //"github.com/prometheus/client_golang/prometheus/push" "github.com/prometheus/common/expfmt" diff --git a/integration-test.sh b/integration-test.sh index a0eda533..afe90c16 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -33,7 +33,7 @@ input: path: $(cygpath -w $log_file) readall: true grok: - patterns_dir: $(cygpath -w $GOPATH/src/github.com/fstab/grok_exporter/logstash-patterns-core/patterns) + patterns_dir: $(cygpath -w $GOPATH/src/github.com/yamingwa/grok_exporter/logstash-patterns-core/patterns) additional_patterns: - 'EXIM_MESSAGE [a-zA-Z ]*' metrics: diff --git a/release.sh b/release.sh index f01e4dc2..4ad657f8 100755 --- a/release.sh +++ b/release.sh @@ -7,16 +7,16 @@ set -e # The Darwin release is built natively, Linux and Windows are built in a Docker container #======================================================================================== -cd $GOPATH/src/github.com/fstab/grok_exporter +cd $GOPATH/src/github.com/yamingwa/grok_exporter rm -rf dist export VERSION=0.2.2-SNAPSHOT export VERSION_FLAGS="\ - -X github.com/fstab/grok_exporter/exporter.Version=$VERSION \ - -X github.com/fstab/grok_exporter/exporter.BuildDate=$(date +%Y-%m-%d) \ - -X github.com/fstab/grok_exporter/exporter.Branch=$(git rev-parse --abbrev-ref HEAD) \ - -X github.com/fstab/grok_exporter/exporter.Revision=$(git rev-parse --short HEAD) \ + -X github.com/yamingwa/grok_exporter/exporter.Version=$VERSION \ + -X github.com/yamingwa/grok_exporter/exporter.BuildDate=$(date +%Y-%m-%d) \ + -X github.com/yamingwa/grok_exporter/exporter.Branch=$(git rev-parse --abbrev-ref HEAD) \ + -X github.com/yamingwa/grok_exporter/exporter.Revision=$(git rev-parse --short HEAD) \ " #-------------------------------------------------------------- @@ -33,7 +33,7 @@ function make_release { echo "Building grok_exporter-$VERSION.$ARCH" mkdir -p dist/grok_exporter-$VERSION.$ARCH if [ $MACHINE = "docker" ] ; then - docker run -v $GOPATH/src/github.com/fstab/grok_exporter:/root/go/src/github.com/fstab/grok_exporter --net none --rm -ti fstab/grok_exporter-compiler compile-$ARCH.sh -ldflags "$VERSION_FLAGS" -o dist/grok_exporter-$VERSION.$ARCH/grok_exporter$EXTENSION + docker run -v $GOPATH/src/github.com/yamingwa/grok_exporter:/root/go/src/github.com/yamingwa/grok_exporter --net none --rm -ti yamingwa/grok_exporter-compiler compile-$ARCH.sh -ldflags "$VERSION_FLAGS" -o dist/grok_exporter-$VERSION.$ARCH/grok_exporter$EXTENSION else # export CGO_LDFLAGS=/usr/local/lib/libonig.a # TODO: For some reason CGO_LDFLAGS does not work on darwin. As a workaround, we set LDFLAGS directly in the header of oniguruma.go. diff --git a/vendor/update.sh b/vendor/update.sh index e26b35d0..76e6aede 100755 --- a/vendor/update.sh +++ b/vendor/update.sh @@ -5,7 +5,7 @@ set -e # patches are created with # diff -Naur proj_orig proj_patched -export VENDOR=$GOPATH/src/github.com/fstab/grok_exporter/vendor +export VENDOR=$GOPATH/src/github.com/yamingwa/grok_exporter/vendor cd $VENDOR # remove all subdirectories From af419d165ce5db19a7e588e2f13266840e6ce666 Mon Sep 17 00:00:00 2001 From: wangyaming Date: Fri, 9 Feb 2018 17:26:52 +0800 Subject: [PATCH 3/3] Revert "change dependencies to github.com/yamingwa/*" This reverts commit a0a08ddb842f54dd3492338adb7bcf5fb9c2dfd9. --- CONFIG.md | 2 +- CONFIG_v1.md | 2 +- README.md | 30 +++++++++++++++--------------- config/config.go | 4 ++-- config/v1/configV1.go | 2 +- config/v2/configV2.go | 2 +- exporter/bufferedTailer.go | 2 +- exporter/grok.go | 2 +- exporter/grok_test.go | 2 +- exporter/metrics.go | 4 ++-- exporter/metrics_test.go | 2 +- grok_exporter.go | 8 ++++---- integration-test.sh | 2 +- release.sh | 12 ++++++------ vendor/update.sh | 2 +- 15 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 76126094..173de77a 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -329,7 +329,7 @@ server: [gauge metric]: https://prometheus.io/docs/concepts/metric_types/#gauge [summary metric]: https://prometheus.io/docs/concepts/metric_types/#summary [histogram metric]: https://prometheus.io/docs/concepts/metric_types/#histogram -[release]: https://github.com/yamingwa/grok_exporter/releases +[release]: https://github.com/fstab/grok_exporter/releases [Prometheus metric types]: https://prometheus.io/docs/concepts/metric_types [Grok documentation]: https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html [histograms and summaries]: https://prometheus.io/docs/practices/histograms/ diff --git a/CONFIG_v1.md b/CONFIG_v1.md index 99da1be2..9bcf135e 100644 --- a/CONFIG_v1.md +++ b/CONFIG_v1.md @@ -296,7 +296,7 @@ server: [gauge metric]: https://prometheus.io/docs/concepts/metric_types/#gauge [summary metric]: https://prometheus.io/docs/concepts/metric_types/#summary [histogram metric]: https://prometheus.io/docs/concepts/metric_types/#histogram -[release]: https://github.com/yamingwa/grok_exporter/releases +[release]: https://github.com/fstab/grok_exporter/releases [Prometheus metric types]: https://prometheus.io/docs/concepts/metric_types [Prometheus data model documentation]: https://prometheus.io/docs/concepts/data_model [Grok documentation]: https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html diff --git a/README.md b/README.md index fee01135..ecbde91e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/yamingwa/grok_exporter.svg?branch=master)](https://travis-ci.org/yamingwa/grok_exporter) [![Build status](https://ci.appveyor.com/api/projects/status/d8aq0pa3yfoapd69?svg=true)](https://ci.appveyor.com/project/yamingwa/grok-exporter) [![Coverage Status](https://coveralls.io/repos/github/yamingwa/grok_exporter/badge.svg?branch=master)](https://coveralls.io/github/yamingwa/grok_exporter?branch=master) +[![Build Status](https://travis-ci.org/fstab/grok_exporter.svg?branch=master)](https://travis-ci.org/fstab/grok_exporter) [![Build status](https://ci.appveyor.com/api/projects/status/d8aq0pa3yfoapd69?svg=true)](https://ci.appveyor.com/project/fstab/grok-exporter) [![Coverage Status](https://coveralls.io/repos/github/fstab/grok_exporter/badge.svg?branch=master)](https://coveralls.io/github/fstab/grok_exporter?branch=master) grok_exporter ============= @@ -61,9 +61,9 @@ Status Operating system support: -* Linux 64 Bit: [Supported](https://travis-ci.org/yamingwa/grok_exporter) -* Windows 64 Bit: [Supported](https://ci.appveyor.com/project/yamingwa/grok-exporter) -* mac OS 64 Bit: [Supported](https://travis-ci.org/yamingwa/grok_exporter) +* Linux 64 Bit: [Supported](https://travis-ci.org/fstab/grok_exporter) +* Windows 64 Bit: [Supported](https://ci.appveyor.com/project/fstab/grok-exporter) +* mac OS 64 Bit: [Supported](https://travis-ci.org/fstab/grok_exporter) Grok pattern support: @@ -86,7 +86,7 @@ In order to compile `grok_exporter` from source, you need [Go] installed and `$G The current version of `brew install oniguruma` will install Oniguruma 6.1.0. Because of [this bug](https://github.com/kkos/oniguruma/issues/23) version 6.1.0 will not work with grok_exporter. Use the following to install the stable 5.9.6 version: ```bash -brew install yamingwa/oniguruma/oniguruma-5.9.6 +brew install fstab/oniguruma/oniguruma-5.9.6 ``` **Installing the Oniguruma library on Ubuntu Linux** @@ -112,8 +112,8 @@ cd onig-5.9.6 && ./configure && make && make install With Oniguruma 5.9.6 installed, download and compile `grok_exporter` as follows: ```bash -go get github.com/yamingwa/grok_exporter -cd $GOPATH/src/github.com/yamingwa/grok_exporter +go get github.com/fstab/grok_exporter +cd $GOPATH/src/github.com/fstab/grok_exporter git submodule update --init --recursive ``` @@ -129,8 +129,8 @@ User documentation is included in the [GitHub repository]: Developer notes are available on the [GitHub Wiki pages]: -* [tailer (tail -f)](https://github.com/yamingwa/grok_exporter/wiki/tailer-(tail-%E2%80%90f)) -* [About the Regular Expression Library](https://github.com/yamingwa/grok_exporter/wiki/About-the-Regular-Expression-Library) +* [tailer (tail -f)](https://github.com/fstab/grok_exporter/wiki/tailer-(tail-%E2%80%90f)) +* [About the Regular Expression Library](https://github.com/fstab/grok_exporter/wiki/About-the-Regular-Expression-Library) External documentation: @@ -143,7 +143,7 @@ Contact * For feature requests, bugs reports, etc: Please open a GitHub issue. * For bug fixes, contributions, etc: Create a pull request. -* Questions? Contact me at fabian@yamingwa.de. +* Questions? Contact me at fabian@fstab.de. Related Projects ---------------- @@ -165,7 +165,7 @@ You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE- [Go]: https://golang.org/ [Oniguruma]: https://github.com/kkos/oniguruma [screenshot.png]: screenshot.png -[releases]: https://github.com/yamingwa/grok_exporter/releases +[releases]: https://github.com/fstab/grok_exporter/releases [http://localhost:9144/metrics]: http://localhost:9144/metrics [CONFIG.md]: CONFIG.md [BUILTIN.md]: BUILTIN.md @@ -177,11 +177,11 @@ You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE- [libpcre]: http://www.pcre.org [rubex]: https://github.com/moovweb/rubex [http://www.apache.org/licenses/LICENSE-2.0]: http://www.apache.org/licenses/LICENSE-2.0 -[more CPU efficient]: https://github.com/yamingwa/grok_exporter/wiki/About-the-Regular-Expression-Library +[more CPU efficient]: https://github.com/fstab/grok_exporter/wiki/About-the-Regular-Expression-Library [fsnotify]: https://github.com/fsnotify/fsnotify -[might be an obstacle]: https://github.com/yamingwa/grok_exporter/wiki/tailer-(tail-%E2%80%90f) -[GitHub Wiki pages]: https://github.com/yamingwa/grok_exporter/wiki -[GitHub repository]: https://github.com/yamingwa/grok_exporter +[might be an obstacle]: https://github.com/fstab/grok_exporter/wiki/tailer-(tail-%E2%80%90f) +[GitHub Wiki pages]: https://github.com/fstab/grok_exporter/wiki +[GitHub repository]: https://github.com/fstab/grok_exporter [Counter]: https://prometheus.io/docs/concepts/metric_types/#counter [Gauge]: https://prometheus.io/docs/concepts/metric_types/#gauge [Histogram]: https://prometheus.io/docs/concepts/metric_types/#histogram diff --git a/config/config.go b/config/config.go index 4fcc431e..7fe890ab 100644 --- a/config/config.go +++ b/config/config.go @@ -16,8 +16,8 @@ package config import ( "fmt" - "github.com/yamingwa/grok_exporter/config/v1" - "github.com/yamingwa/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/config/v1" + "github.com/fstab/grok_exporter/config/v2" "io/ioutil" "regexp" "strconv" diff --git a/config/v1/configV1.go b/config/v1/configV1.go index 61781fb5..1ca9aa51 100644 --- a/config/v1/configV1.go +++ b/config/v1/configV1.go @@ -16,7 +16,7 @@ package v1 import ( "fmt" - "github.com/yamingwa/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/config/v2" "gopkg.in/yaml.v2" ) diff --git a/config/v2/configV2.go b/config/v2/configV2.go index d51acad1..5466ac9f 100644 --- a/config/v2/configV2.go +++ b/config/v2/configV2.go @@ -16,7 +16,7 @@ package v2 import ( "fmt" - "github.com/yamingwa/grok_exporter/templates" + "github.com/fstab/grok_exporter/templates" "gopkg.in/yaml.v2" "regexp" ) diff --git a/exporter/bufferedTailer.go b/exporter/bufferedTailer.go index 52678c18..fa69d161 100644 --- a/exporter/bufferedTailer.go +++ b/exporter/bufferedTailer.go @@ -16,7 +16,7 @@ package exporter import ( "container/list" - "github.com/yamingwa/grok_exporter/tailer" + "github.com/fstab/grok_exporter/tailer" "github.com/prometheus/client_golang/prometheus" "log" "sync" diff --git a/exporter/grok.go b/exporter/grok.go index 14c8cc56..65121549 100644 --- a/exporter/grok.go +++ b/exporter/grok.go @@ -16,7 +16,7 @@ package exporter import ( "fmt" - "github.com/yamingwa/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/config/v2" "regexp" "strings" ) diff --git a/exporter/grok_test.go b/exporter/grok_test.go index 581fb622..c12d9b19 100644 --- a/exporter/grok_test.go +++ b/exporter/grok_test.go @@ -15,7 +15,7 @@ package exporter import ( - "github.com/yamingwa/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/config/v2" "gopkg.in/yaml.v2" "strings" "testing" diff --git a/exporter/metrics.go b/exporter/metrics.go index 915cb567..409d43a9 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -16,8 +16,8 @@ package exporter import ( "fmt" - "github.com/yamingwa/grok_exporter/config/v2" - "github.com/yamingwa/grok_exporter/templates" + "github.com/fstab/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/templates" "github.com/prometheus/client_golang/prometheus" "strconv" ) diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index eb555cfb..83d313b1 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -15,7 +15,7 @@ package exporter import ( - "github.com/yamingwa/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/config/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_model/go" "reflect" diff --git a/grok_exporter.go b/grok_exporter.go index 2f3e348a..33153a14 100644 --- a/grok_exporter.go +++ b/grok_exporter.go @@ -18,10 +18,10 @@ import ( "bytes" "flag" "fmt" - "github.com/yamingwa/grok_exporter/config" - "github.com/yamingwa/grok_exporter/config/v2" - "github.com/yamingwa/grok_exporter/exporter" - "github.com/yamingwa/grok_exporter/tailer" + "github.com/fstab/grok_exporter/config" + "github.com/fstab/grok_exporter/config/v2" + "github.com/fstab/grok_exporter/exporter" + "github.com/fstab/grok_exporter/tailer" "github.com/prometheus/client_golang/prometheus" //"github.com/prometheus/client_golang/prometheus/push" "github.com/prometheus/common/expfmt" diff --git a/integration-test.sh b/integration-test.sh index afe90c16..a0eda533 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -33,7 +33,7 @@ input: path: $(cygpath -w $log_file) readall: true grok: - patterns_dir: $(cygpath -w $GOPATH/src/github.com/yamingwa/grok_exporter/logstash-patterns-core/patterns) + patterns_dir: $(cygpath -w $GOPATH/src/github.com/fstab/grok_exporter/logstash-patterns-core/patterns) additional_patterns: - 'EXIM_MESSAGE [a-zA-Z ]*' metrics: diff --git a/release.sh b/release.sh index 4ad657f8..f01e4dc2 100755 --- a/release.sh +++ b/release.sh @@ -7,16 +7,16 @@ set -e # The Darwin release is built natively, Linux and Windows are built in a Docker container #======================================================================================== -cd $GOPATH/src/github.com/yamingwa/grok_exporter +cd $GOPATH/src/github.com/fstab/grok_exporter rm -rf dist export VERSION=0.2.2-SNAPSHOT export VERSION_FLAGS="\ - -X github.com/yamingwa/grok_exporter/exporter.Version=$VERSION \ - -X github.com/yamingwa/grok_exporter/exporter.BuildDate=$(date +%Y-%m-%d) \ - -X github.com/yamingwa/grok_exporter/exporter.Branch=$(git rev-parse --abbrev-ref HEAD) \ - -X github.com/yamingwa/grok_exporter/exporter.Revision=$(git rev-parse --short HEAD) \ + -X github.com/fstab/grok_exporter/exporter.Version=$VERSION \ + -X github.com/fstab/grok_exporter/exporter.BuildDate=$(date +%Y-%m-%d) \ + -X github.com/fstab/grok_exporter/exporter.Branch=$(git rev-parse --abbrev-ref HEAD) \ + -X github.com/fstab/grok_exporter/exporter.Revision=$(git rev-parse --short HEAD) \ " #-------------------------------------------------------------- @@ -33,7 +33,7 @@ function make_release { echo "Building grok_exporter-$VERSION.$ARCH" mkdir -p dist/grok_exporter-$VERSION.$ARCH if [ $MACHINE = "docker" ] ; then - docker run -v $GOPATH/src/github.com/yamingwa/grok_exporter:/root/go/src/github.com/yamingwa/grok_exporter --net none --rm -ti yamingwa/grok_exporter-compiler compile-$ARCH.sh -ldflags "$VERSION_FLAGS" -o dist/grok_exporter-$VERSION.$ARCH/grok_exporter$EXTENSION + docker run -v $GOPATH/src/github.com/fstab/grok_exporter:/root/go/src/github.com/fstab/grok_exporter --net none --rm -ti fstab/grok_exporter-compiler compile-$ARCH.sh -ldflags "$VERSION_FLAGS" -o dist/grok_exporter-$VERSION.$ARCH/grok_exporter$EXTENSION else # export CGO_LDFLAGS=/usr/local/lib/libonig.a # TODO: For some reason CGO_LDFLAGS does not work on darwin. As a workaround, we set LDFLAGS directly in the header of oniguruma.go. diff --git a/vendor/update.sh b/vendor/update.sh index 76e6aede..e26b35d0 100755 --- a/vendor/update.sh +++ b/vendor/update.sh @@ -5,7 +5,7 @@ set -e # patches are created with # diff -Naur proj_orig proj_patched -export VENDOR=$GOPATH/src/github.com/yamingwa/grok_exporter/vendor +export VENDOR=$GOPATH/src/github.com/fstab/grok_exporter/vendor cd $VENDOR # remove all subdirectories