diff --git a/config/v2/configV2.go b/config/v2/configV2.go index 0804081d..5466ac9f 100644 --- a/config/v2/configV2.go +++ b/config/v2/configV2.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/fstab/grok_exporter/templates" "gopkg.in/yaml.v2" + "regexp" ) func Unmarshal(config []byte) (*Config, error) { @@ -42,7 +43,8 @@ func (cfg *Config) String() string { } type GlobalConfig struct { - ConfigVersion int `yaml:"config_version,omitempty"` + ConfigVersion int `yaml:"config_version,omitempty"` + PushgatewayAddr string `yaml:"pushgateway_addr,omitempty"` // add pushgateway address : } type InputConfig struct { @@ -57,10 +59,19 @@ type GrokConfig struct { } type MetricConfig struct { - Type string `yaml:",omitempty"` - Name string `yaml:",omitempty"` - Help string `yaml:",omitempty"` - Match string `yaml:",omitempty"` + Type string `yaml:",omitempty"` + Name string `yaml:",omitempty"` + Help string `yaml:",omitempty"` + Match string `yaml:",omitempty"` + + /**************pushgateway related configs*******************/ + Pushgateway bool `yaml:",omitempty"` //flag for if push metric to pushgateway or not + JobName string `yaml:"job_name,omitempty"` //job name used to push metric to pushgateway + DeleteMatch string `yaml:"delete_match,omitempty"` //if match, metric will be deleted from pushgateway + GroupingKey map[string]string `yaml:"grouping_key,omitempty"` //grouping key used to push and delete metric from pushgateway + GroupTemplates []templates.Template `yaml:"-"` + /**************end of pushgateway related configs************/ + Value string `yaml:",omitempty"` Cumulative bool `yaml:",omitempty"` Buckets []float64 `yaml:",flow,omitempty"` @@ -138,7 +149,11 @@ func (c *ServerConfig) addDefaults() { } func (cfg *Config) validate() error { - err := cfg.Input.validate() + err := cfg.Global.validate() + if err != nil { + return err + } + err = cfg.Input.validate() if err != nil { return err } @@ -157,6 +172,17 @@ func (cfg *Config) validate() error { return nil } +func (c *GlobalConfig) validate() error { + //ignore version validation + if len(c.PushgatewayAddr) > 0 { + reg := regexp.MustCompile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:[0-9]{1,5}") + result := reg.FindString(c.PushgatewayAddr) + if result == "" { + return fmt.Errorf("Not valid pushgateway address, usage: >:.") + } + } + return nil +} func (c *InputConfig) validate() error { switch { case c.Type == "stdin": @@ -236,6 +262,11 @@ func (c *MetricConfig) validate() error { return fmt.Errorf("Invalid metric configuration: 'metrics.buckets' cannot be used for %v metrics.", c.Type) } // Labels and value are validated in InitTemplates() + //validate pushgateway related configs + if len(c.DeleteMatch) > 0 && !c.Pushgateway { + return fmt.Errorf("Invalid metric configuration: 'delete_match' can only be defined when 'pushgateway = true'.") + } + //grouping key is validated in InitTemplates() return nil } @@ -289,6 +320,16 @@ func (metric *MetricConfig) InitTemplates() error { } metric.LabelTemplates = append(metric.LabelTemplates, tmplt) } + // validate grouping key + metric.GroupTemplates = make([]templates.Template, 0, len(metric.GroupingKey)) + for name, templateString := range metric.GroupingKey { + tmplt, err = templates.New(name, templateString) + if err != nil { + return fmt.Errorf(msg, fmt.Sprintf("groupingKey %v", metric.Name), name, err.Error()) + } + metric.GroupTemplates = append(metric.GroupTemplates, tmplt) + } + if len(metric.Value) > 0 { metric.ValueTemplate, err = templates.New("__value__", metric.Value) if err != nil { diff --git a/exporter/grok.go b/exporter/grok.go index a2e3fc48..65121549 100644 --- a/exporter/grok.go +++ b/exporter/grok.go @@ -52,6 +52,18 @@ func VerifyFieldNames(m *v2.MetricConfig, regex *OnigurumaRegexp) error { return nil } +func VerifyGroupingKeyField(m *v2.MetricConfig, regex *OnigurumaRegexp) error { + for _, template := range m.GroupTemplates { + for _, grokFieldName := range template.ReferencedGrokFields() { + if !regex.HasCaptureGroup(grokFieldName) { + return fmt.Errorf("%v: error in label %v: grok field %v not found in match pattern", m.Name, template.Name(), grokFieldName) + } + } + } + + return nil +} + // PATTERN_RE matches the %{..} patterns. There are three possibilities: // 1) %{USER} - grok pattern // 2) %{IP:clientip} - grok pattern with name diff --git a/exporter/metrics.go b/exporter/metrics.go index edb6e1d8..409d43a9 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -25,9 +25,12 @@ import ( type Metric interface { Name() string Collector() prometheus.Collector + MetricVec() *prometheus.MetricVec // Returns true if the line matched, and false if the line didn't match. - Process(line string) (bool, error) + Process(line string) (bool, bool, map[string]string, []string, error) + NeedPush() bool + JobName() string } // Represents a Prometheus Counter @@ -36,20 +39,34 @@ type incMetric struct { regex *OnigurumaRegexp labels []templates.Template collector prometheus.Collector - incFunc func(m *OnigurumaMatchResult) error + metricVec *prometheus.MetricVec + //pushgateway related configs + delete_regex *OnigurumaRegexp + pushgateway bool + job_name string + groupingKey []templates.Template + + incFunc func(m *OnigurumaMatchResult) error } // Represents a Prometheus Gauge, Histogram, or Summary type observeMetric struct { - name string - regex *OnigurumaRegexp - value templates.Template - labels []templates.Template + name string + regex *OnigurumaRegexp + value templates.Template + labels []templates.Template + //pushgateway related configs + delete_regex *OnigurumaRegexp + pushgateway bool + job_name string + groupingKey []templates.Template + collector prometheus.Collector + metricVec *prometheus.MetricVec observeFunc func(m *OnigurumaMatchResult, val float64) error } -func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { counterOpts := prometheus.CounterOpts{ Name: cfg.Name, Help: cfg.Help, @@ -57,9 +74,13 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular counter counter := prometheus.NewCounter(counterOpts) return &incMetric{ - name: cfg.Name, - regex: regex, - collector: counter, + name: cfg.Name, + regex: regex, + collector: counter, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, incFunc: func(_ *OnigurumaMatchResult) error { counter.Inc() return nil @@ -68,10 +89,15 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // counterVec counterVec := prometheus.NewCounterVec(counterOpts, prometheusLabels(cfg.LabelTemplates)) result := &incMetric{ - name: cfg.Name, - regex: regex, - labels: cfg.LabelTemplates, - collector: counterVec, + name: cfg.Name, + regex: regex, + labels: cfg.LabelTemplates, + collector: counterVec, + metricVec: counterVec.MetricVec, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, incFunc: func(m *OnigurumaMatchResult) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -84,7 +110,7 @@ func NewCounterMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } } -func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { gaugeOpts := prometheus.GaugeOpts{ Name: cfg.Name, Help: cfg.Help, @@ -92,10 +118,14 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular gauge gauge := prometheus.NewGauge(gaugeOpts) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: gauge, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: gauge, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { if cfg.Cumulative { gauge.Add(val) @@ -108,11 +138,16 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // gaugeVec gaugeVec := prometheus.NewGaugeVec(gaugeOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: gaugeVec, - labels: cfg.LabelTemplates, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: gaugeVec, + metricVec: gaugeVec.MetricVec, + labels: cfg.LabelTemplates, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -128,7 +163,7 @@ func NewGaugeMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } } -func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { histogramOpts := prometheus.HistogramOpts{ Name: cfg.Name, Help: cfg.Help, @@ -139,10 +174,14 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular histogram histogram := prometheus.NewHistogram(histogramOpts) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: histogram, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: histogram, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { histogram.Observe(val) return nil @@ -151,11 +190,16 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // histogramVec histogramVec := prometheus.NewHistogramVec(histogramOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: histogramVec, - labels: cfg.LabelTemplates, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: histogramVec, + metricVec: histogramVec.MetricVec, + labels: cfg.LabelTemplates, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -167,7 +211,7 @@ func NewHistogramMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } } -func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { +func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp, delete_regex *OnigurumaRegexp) Metric { summaryOpts := prometheus.SummaryOpts{ Name: cfg.Name, Help: cfg.Help, @@ -178,10 +222,14 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { if len(cfg.Labels) == 0 { // regular summary summary := prometheus.NewSummary(summaryOpts) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: summary, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: summary, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(_ *OnigurumaMatchResult, val float64) error { summary.Observe(val) return nil @@ -190,11 +238,16 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } else { // summaryVec summaryVec := prometheus.NewSummaryVec(summaryOpts, prometheusLabels(cfg.LabelTemplates)) return &observeMetric{ - name: cfg.Name, - regex: regex, - value: cfg.ValueTemplate, - collector: summaryVec, - labels: cfg.LabelTemplates, + name: cfg.Name, + regex: regex, + value: cfg.ValueTemplate, + collector: summaryVec, + metricVec: summaryVec.MetricVec, + labels: cfg.LabelTemplates, + delete_regex: delete_regex, + pushgateway: cfg.Pushgateway, + job_name: cfg.JobName, + groupingKey: cfg.GroupTemplates, observeFunc: func(m *OnigurumaMatchResult, val float64) error { vals, err := labelValues(m, cfg.LabelTemplates) if err == nil { @@ -207,40 +260,91 @@ func NewSummaryMetric(cfg *v2.MetricConfig, regex *OnigurumaRegexp) Metric { } // Return: true if the line matched, false if it didn't match. -func (m *incMetric) Process(line string) (bool, error) { +func (m *incMetric) Process(line string) (bool, bool, map[string]string, []string, error) { matchResult, err := m.regex.Match(line) if err != nil { - return false, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + } + var deleteMatch *OnigurumaMatchResult = nil + var e error + if m.delete_regex != nil { + deleteMatch, e = m.delete_regex.Match(line) + if e != nil { + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, e.Error()) + } } + defer matchResult.Free() + if deleteMatch != nil { + defer deleteMatch.Free() + } + + //metric can either be pushed or deleted, CANNOT be both in single line processing if matchResult.IsMatch() { err = m.incFunc(matchResult) - return true, err + metricLabelValues, err := labelValues(matchResult, m.labels) + groupingKey, e := evalGroupingKey(matchResult, m.groupingKey) + if e != nil { + return true, false, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, e.Error()) + } + return true, false, groupingKey, metricLabelValues, err + } else { - return false, nil + if deleteMatch != nil && deleteMatch.IsMatch() { + groupingKey, e := evalGroupingKey(deleteMatch, m.groupingKey) + if e != nil { + return false, true, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, e.Error()) + } + return false, true, groupingKey, nil, nil + } + return false, false, nil, nil, nil } } // Return: true if the line matched, false if it didn't match. -func (m *observeMetric) Process(line string) (bool, error) { +func (m *observeMetric) Process(line string) (bool, bool, map[string]string, []string, error) { matchResult, err := m.regex.Match(line) if err != nil { - return false, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + } + var deleteMatch *OnigurumaMatchResult = nil + var e error + if m.delete_regex != nil { + deleteMatch, e = m.delete_regex.Match(line) + if e != nil { + return false, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, e.Error()) + } } defer matchResult.Free() + if deleteMatch != nil { + defer deleteMatch.Free() + } + if matchResult.IsMatch() { stringVal, err := evalTemplate(matchResult, m.value) if err != nil { - return true, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) + return true, false, nil, nil, fmt.Errorf("error while processing metric %v: %v", m.name, err.Error()) } floatVal, err := strconv.ParseFloat(stringVal, 64) if err != nil { - return true, fmt.Errorf("error while processing metric %v: value '%v' matches '%v', which is not a valid number.", m.name, m.value, stringVal) + return true, false, nil, nil, fmt.Errorf("error while processing metric %v: value '%v' matches '%v', which is not a valid number.", m.name, m.value, stringVal) } err = m.observeFunc(matchResult, floatVal) - return true, err + metricLabelValues, err := labelValues(matchResult, m.labels) + groupingKey, e := evalGroupingKey(matchResult, m.groupingKey) + if e != nil { + return true, false, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, e.Error()) + } + return true, false, groupingKey, metricLabelValues, err } else { - return false, nil + if deleteMatch != nil && deleteMatch.IsMatch() { + groupingKey, err := evalGroupingKey(deleteMatch, m.groupingKey) + if err != nil { + return false, true, nil, nil, fmt.Errorf("error while getting grouping key %v: %v", m.name, err.Error()) + } + return false, true, groupingKey, nil, nil + } + return false, false, nil, nil, nil } } @@ -260,6 +364,52 @@ func (m *observeMetric) Collector() prometheus.Collector { return m.collector } +func (m *incMetric) MetricVec() *prometheus.MetricVec { + return m.metricVec +} + +func (m *observeMetric) MetricVec() *prometheus.MetricVec { + return m.metricVec +} + +func (m *incMetric) NeedPush() bool { + return m.pushgateway +} + +func (m *observeMetric) NeedPush() bool { + return m.pushgateway +} + +func (m *incMetric) JobName() string { + if len(m.job_name) != 0 { + return m.job_name + } else { + return "grok_exporter" + } +} + +func (m *observeMetric) JobName() string { + if len(m.job_name) != 0 { + return m.job_name + } else { + return "grok_exporter" + } +} + +func evalGroupingKey(matchResult *OnigurumaMatchResult, templates []templates.Template) (map[string]string, error) { + result := make(map[string]string, len(templates)) + for _, t := range templates { + value, err := evalTemplate(matchResult, t) + if err != nil { + return nil, err + } + result[t.Name()] = value + + } + //fmt.Println("[DEBUG] got groupingKey: %s", result) + return result, nil +} + func labelValues(matchResult *OnigurumaMatchResult, templates []templates.Template) ([]string, error) { result := make([]string, 0, len(templates)) for _, t := range templates { diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index c54deab8..83d313b1 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -30,7 +30,7 @@ func TestCounterVec(t *testing.T) { "error_message": "{{.message}}", }, }) - counter := NewCounterMetric(counterCfg, regex) + counter := NewCounterMetric(counterCfg, regex, nil) counter.Process("some unrelated line") counter.Process("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F= rejected RCPT : relay not permitted") counter.Process("2016-04-26 12:31:39 H=(186-90-8-31.genericrev.cantv.net) [186.90.8.31] F= rejected RCPT : Unrouteable address") @@ -57,7 +57,7 @@ func TestCounter(t *testing.T) { counterCfg := newMetricConfig(t, &v2.MetricConfig{ Name: "exim_rejected_rcpt_total", }) - counter := NewCounterMetric(counterCfg, regex) + counter := NewCounterMetric(counterCfg, regex, nil) counter.Process("some unrelated line") counter.Process("2016-04-26 10:19:57 H=(85.214.241.101) [36.224.138.227] F= rejected RCPT : relay not permitted") @@ -99,7 +99,7 @@ func TestGauge(t *testing.T) { Name: "temperature", Value: "{{.temperature}}", }) - gauge := NewGaugeMetric(gaugeCfg, regex) + gauge := NewGaugeMetric(gaugeCfg, regex, nil) gauge.Process("Temperature in Berlin: 32") gauge.Process("Temperature in Moscow: -5") @@ -123,7 +123,7 @@ func TestGaugeCumulative(t *testing.T) { Value: "{{.temperature}}", Cumulative: true, }) - gauge := NewGaugeMetric(gaugeCfg, regex) + gauge := NewGaugeMetric(gaugeCfg, regex, nil) gauge.Process("Temperature in Berlin: 32") gauge.Process("Temperature in Moscow: -5") @@ -149,7 +149,7 @@ func TestGaugeVec(t *testing.T) { "city": "{{.city}}", }, }) - gauge := NewGaugeMetric(gaugeCfg, regex) + gauge := NewGaugeMetric(gaugeCfg, regex, nil) gauge.Process("Temperature in Berlin: 32") gauge.Process("Temperature in Moscow: -5") diff --git a/grok_exporter.go b/grok_exporter.go index f32799d5..33153a14 100644 --- a/grok_exporter.go +++ b/grok_exporter.go @@ -15,6 +15,7 @@ package main import ( + "bytes" "flag" "fmt" "github.com/fstab/grok_exporter/config" @@ -22,8 +23,13 @@ import ( "github.com/fstab/grok_exporter/exporter" "github.com/fstab/grok_exporter/tailer" "github.com/prometheus/client_golang/prometheus" + //"github.com/prometheus/client_golang/prometheus/push" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" "net/http" + "net/url" "os" + "strings" "time" ) @@ -68,7 +74,7 @@ func main() { tail, err := startTailer(cfg) exitOnError(err) - fmt.Print(startMsg(cfg)) + fmt.Println(startMsg(cfg)) serverErrors := startServer(cfg, "/metrics", prometheus.Handler()) for { @@ -81,17 +87,35 @@ func main() { matched := false for _, metric := range metrics { start := time.Now() - match, err := metric.Process(line) + match, delete_match, groupingKey, labelValues, err := metric.Process(line) + //fmt.Println(fmt.Sprintf("[DEBUG] Process result: match: %s, delete_match: %s, groupingKey: %s, err: %s", match, delete_match, groupingKey, err)) + + pushFlag := true if err != nil { fmt.Fprintf(os.Stderr, "WARNING: Skipping log line: %v\n", err.Error()) fmt.Fprintf(os.Stderr, "%v\n", line) nErrorsByMetric.WithLabelValues(metric.Name()).Inc() + pushFlag = false } if match { + if metric.NeedPush() && pushFlag { + err := pushMetric(metric, cfg.Global.PushgatewayAddr, groupingKey, labelValues) + if err != nil { + //fmt.Println(fmt.Sprintf("[DEBUG] Push error: %s", err)) + fmt.Errorf("Error pushing metric %v to pushgateway.", metric.Name()) + } + } + nMatchesByMetric.WithLabelValues(metric.Name()).Inc() procTimeMicrosecondsByMetric.WithLabelValues(metric.Name()).Add(float64(time.Since(start).Nanoseconds() / int64(1000))) matched = true } + if delete_match { + err := deleteMetric(metric, cfg.Global.PushgatewayAddr, groupingKey) + if err != nil { + fmt.Errorf("Error deleting metric %v from pushgateway.", metric.Name()) + } + } } if matched { nLinesTotal.WithLabelValues(number_of_lines_matched_label).Inc() @@ -102,6 +126,89 @@ func main() { } } +func pushMetric(m exporter.Metric, pushUrl string, groupingKey map[string]string, labelValues []string) error { + //fmt.Println(fmt.Sprintf("[DEBUG] Pushing metric %s with labels %s to pushgateway %s of job %s", m.Name(), groupingKey, pushUrl, m.JobName())) + r := prometheus.NewRegistry() + if err := r.Register(m.Collector()); err != nil { + return err + } + err := doRequest(m.JobName(), groupingKey, pushUrl, r, "POST") + if err != nil { + return err + } + //remove metric from collector + if m.MetricVec() != nil { + m.MetricVec().DeleteLabelValues(labelValues...) + } + return nil +} + +func deleteMetric(m exporter.Metric, deleteUrl string, groupingKey map[string]string) error { + //fmt.Println(fmt.Sprintf("[DEBUG] Deleting metric %s with labels %s from pushgateway %s of job %s", m.Name(), groupingKey, deleteUrl, m.JobName())) + return doRequest(m.JobName(), groupingKey, deleteUrl, nil, "DELETE") + +} + +func doRequest(job string, groupingKey map[string]string, targetUrl string, g prometheus.Gatherer, method string) error { + if !strings.Contains(targetUrl, "://") { + targetUrl = "http://" + targetUrl + } + if strings.HasSuffix(targetUrl, "/") { + targetUrl = targetUrl[:len(targetUrl)-1] + } + + if strings.Contains(job, "/") { + return fmt.Errorf("job contains '/' : %s", job) + } + urlComponents := []string{url.QueryEscape(job)} + for ln, lv := range groupingKey { + if !model.LabelName(ln).IsValid() { + return fmt.Errorf("groupingKey label has invalid name: %s", ln) + } + if strings.Contains(lv, "/") { + return fmt.Errorf("value of groupingKey label %s contains '/': %s", ln, lv) + } + urlComponents = append(urlComponents, ln, lv) + } + + targetUrl = fmt.Sprintf("%s/metrics/job/%s", targetUrl, strings.Join(urlComponents, "/")) + + buf := &bytes.Buffer{} + enc := expfmt.NewEncoder(buf, expfmt.FmtProtoDelim) + if g != nil { + mfs, err := g.Gather() + if err != nil { + return err + } + for _, mf := range mfs { + //ignore checking for pre-existing labels + enc.Encode(mf) + } + } + + var request *http.Request + var err error + if method == "DELETE" { + request, err = http.NewRequest(method, targetUrl, nil) + } else { + request, err = http.NewRequest(method, targetUrl, buf) + } + + if err != nil { + return err + } + request.Header.Set("Content-Type", string(expfmt.FmtProtoDelim)) + response, err := http.DefaultClient.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode != 202 { + return fmt.Errorf("unexpected status code %d, method %s", response.StatusCode, method) + } + return nil +} + func startMsg(cfg *v2.Config) string { host := "localhost" if len(cfg.Server.Host) > 0 { @@ -161,18 +268,33 @@ func createMetrics(cfg *v2.Config, patterns *exporter.Patterns, libonig *exporte if err != nil { return nil, fmt.Errorf("failed to initialize metric %v: %v", m.Name, err.Error()) } + + var delete_regex *exporter.OnigurumaRegexp = nil + + if len(m.DeleteMatch) != 0 { + delete_regex, err = exporter.Compile(m.DeleteMatch, patterns, libonig) + if err != nil { + return nil, fmt.Errorf("failed to initialize metric %v: %v", m.Name, err.Error()) + } + err = exporter.VerifyGroupingKeyField(m, delete_regex) + if err != nil { + return nil, fmt.Errorf("failed to initialize metric %v: %v", m.Name, err.Error()) + } + } + switch m.Type { case "counter": - result = append(result, exporter.NewCounterMetric(m, regex)) + result = append(result, exporter.NewCounterMetric(m, regex, delete_regex)) case "gauge": - result = append(result, exporter.NewGaugeMetric(m, regex)) + result = append(result, exporter.NewGaugeMetric(m, regex, delete_regex)) case "histogram": - result = append(result, exporter.NewHistogramMetric(m, regex)) + result = append(result, exporter.NewHistogramMetric(m, regex, delete_regex)) case "summary": - result = append(result, exporter.NewSummaryMetric(m, regex)) + result = append(result, exporter.NewSummaryMetric(m, regex, delete_regex)) default: return nil, fmt.Errorf("Failed to initialize metrics: Metric type %v is not supported.", m.Type) } + } return result, nil }