diff --git a/examples/grafana/detail.json b/examples/grafana/detail.json index 80da5b35e..b8dfd4bca 100644 --- a/examples/grafana/detail.json +++ b/examples/grafana/detail.json @@ -305,7 +305,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((pyrra_availability{slo=\"$slo\"} - pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})", + "expr": "((pyrra_availability{slo=\"$slo\"} - on(slo) pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -405,7 +405,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "((pyrra_availability{slo=\"$slo\"} - pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})", + "expr": "((pyrra_availability{slo=\"$slo\"} - on(slo) pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})", "legendFormat": "__auto", "range": true, "refId": "A" diff --git a/examples/kubernetes/manifests/setup/pyrra-slo-CustomResourceDefinition.yaml b/examples/kubernetes/manifests/setup/pyrra-slo-CustomResourceDefinition.yaml index 579f3c873..86964d610 100644 --- a/examples/kubernetes/manifests/setup/pyrra-slo-CustomResourceDefinition.yaml +++ b/examples/kubernetes/manifests/setup/pyrra-slo-CustomResourceDefinition.yaml @@ -185,6 +185,9 @@ spec: It represents the desired availability of the service in the given window. float64 are not supported: https://github.com/kubernetes-sigs/controller-tools/issues/245 type: string + rulenamespace: + description: Recording rule namespace override, by default recording namespace and group are kept same. + type: string window: description: Window within which the Target is supposed to be kept. Usually something like 1d, 7d or 28d. type: string diff --git a/go.mod b/go.mod index 1d79f7b8c..e3526ec69 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,9 @@ module github.com/pyrra-dev/pyrra + go 1.23 + require ( github.com/alecthomas/kong v0.9.0 github.com/bufbuild/connect-go v1.10.0 diff --git a/jsonnet/controller-gen/pyrra.dev_servicelevelobjectives.json b/jsonnet/controller-gen/pyrra.dev_servicelevelobjectives.json index 049a2f3b1..11177ce3a 100644 --- a/jsonnet/controller-gen/pyrra.dev_servicelevelobjectives.json +++ b/jsonnet/controller-gen/pyrra.dev_servicelevelobjectives.json @@ -232,6 +232,10 @@ }, "type": "object" }, + "rulenamespace": { + "description": "Recording rule namespace override, by default recording namespace and group are kept same.", + "type": "string" + }, "target": { "description": "Target is a string that's casted to a float64 between 0 - 100.\nIt represents the desired availability of the service in the given window.\nfloat64 are not supported: https://github.com/kubernetes-sigs/controller-tools/issues/245", "type": "string" diff --git a/kubernetes/api/v1alpha1/servicelevelobjective_types.go b/kubernetes/api/v1alpha1/servicelevelobjective_types.go index 349a9c213..c2b863073 100644 --- a/kubernetes/api/v1alpha1/servicelevelobjective_types.go +++ b/kubernetes/api/v1alpha1/servicelevelobjective_types.go @@ -90,6 +90,10 @@ type ServiceLevelObjectiveSpec struct { // +optional // Alerting customizes the alerting rules generated by Pyrra. Alerting Alerting `json:"alerting"` + + // +optional + // Recording rule namespace override, by default recording namespace and group are kept same. + RecordingRuleNamespace string `json:"rulenamespace"` } // ServiceLevelIndicator defines the underlying indicator that is a Prometheus metric. @@ -558,13 +562,14 @@ func (in *ServiceLevelObjective) Internal() (slo.Objective, error) { } return slo.Objective{ - Labels: ls, - Annotations: in.Annotations, - Description: in.Spec.Description, - Target: target / 100, - Window: window, - Config: string(config), - Alerting: alerting, + Labels: ls, + Annotations: in.Annotations, + Description: in.Spec.Description, + Target: target / 100, + Window: window, + Config: string(config), + Alerting: alerting, + RecordingRuleNamespace: in.Spec.RecordingRuleNamespace, Indicator: slo.Indicator{ Ratio: ratio, Latency: latency, diff --git a/kubernetes/api/v1alpha1/servicelevelobjective_types_test.go b/kubernetes/api/v1alpha1/servicelevelobjective_types_test.go index 0564efc03..d2cf0a536 100644 --- a/kubernetes/api/v1alpha1/servicelevelobjective_types_test.go +++ b/kubernetes/api/v1alpha1/servicelevelobjective_types_test.go @@ -56,6 +56,7 @@ spec: Burnrates: true, Absent: true, }, + RecordingRuleNamespace: "", Indicator: slo.Indicator{ Ratio: &slo.RatioIndicator{ Errors: slo.Metric{ @@ -90,6 +91,7 @@ metadata: spec: target: 99.9 window: 1w + rulenamespace: "test-namespace" indicator: ratio: errors: @@ -109,6 +111,7 @@ spec: Burnrates: true, Absent: true, }, + RecordingRuleNamespace: "test-namespace", Indicator: slo.Indicator{ Ratio: &slo.RatioIndicator{ Errors: slo.Metric{ diff --git a/kubernetes/controllers/servicelevelobjective.go b/kubernetes/controllers/servicelevelobjective.go index cf222560d..8614cc459 100644 --- a/kubernetes/controllers/servicelevelobjective.go +++ b/kubernetes/controllers/servicelevelobjective.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "reflect" "time" kitlog "github.com/go-kit/log" @@ -35,6 +36,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/yaml" pyrrav1alpha1 "github.com/pyrra-dev/pyrra/kubernetes/api/v1alpha1" @@ -125,7 +128,6 @@ func (r *ServiceLevelObjectiveReconciler) reconcilePrometheusRule(ctx context.Co } newRule.ResourceVersion = rule.ResourceVersion - level.Info(logger).Log("msg", "updating prometheus rule", "namespace", rule.GetNamespace(), "name", rule.GetName()) if err := r.Update(ctx, newRule); err != nil { return ctrl.Result{}, fmt.Errorf("failed to update prometheus rule: %w", err) @@ -147,7 +149,13 @@ func (r *ServiceLevelObjectiveReconciler) reconcileMimirRuleGroup(ctx context.Co level.Info(logger).Log("msg", "updating mimir rule", "name", newRuleGroup.Name) - err = r.MimirClient.SetRuleGroup(ctx, kubeObjective.GetName(), *newRuleGroup) + var ruleName string + if kubeObjective.Spec.RecordingRuleNamespace == "" { + ruleName = kubeObjective.GetName() + } else { + ruleName = kubeObjective.Spec.RecordingRuleNamespace + } + err = r.MimirClient.SetRuleGroup(ctx, ruleName, *newRuleGroup) if err != nil { return ctrl.Result{}, err } @@ -160,8 +168,15 @@ func (r *ServiceLevelObjectiveReconciler) reconcileMimirRuleGroup(ctx context.Co return ctrl.Result{}, nil } + func (r *ServiceLevelObjectiveReconciler) deleteMimirRuleGroup(ctx context.Context, kubeObjective pyrrav1alpha1.ServiceLevelObjective) error { - return r.MimirClient.DeleteNamespace(ctx, kubeObjective.GetName()) + var ruleName string + if kubeObjective.Spec.RecordingRuleNamespace == "" { + ruleName = kubeObjective.GetName() + } else { + ruleName = kubeObjective.Spec.RecordingRuleNamespace + } + return r.MimirClient.DeleteNamespace(ctx, ruleName) } func (r *ServiceLevelObjectiveReconciler) reconcileConfigMap( @@ -210,6 +225,29 @@ func (r *ServiceLevelObjectiveReconciler) reconcileConfigMap( func (r *ServiceLevelObjectiveReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&pyrrav1alpha1.ServiceLevelObjective{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldObject := e.ObjectOld.(*pyrrav1alpha1.ServiceLevelObjective) + newObject := e.ObjectNew.(*pyrrav1alpha1.ServiceLevelObjective) + + // Check if there's a difference; if not, skip reconciliation + if !reflect.DeepEqual(oldObject.Spec.RecordingRuleNamespace, newObject.Spec.RecordingRuleNamespace) { + + var ruleNamespace string + if oldObject.Spec.RecordingRuleNamespace == "" { + ruleNamespace = oldObject.GetName() + } else { + ruleNamespace = oldObject.Spec.RecordingRuleNamespace + } + + if err := r.MimirClient.DeleteNamespace(context.Background(), ruleNamespace); err != nil { + return false + } + return true + } + return true + }, + }). Complete(r) } diff --git a/main.go b/main.go index 8dad754a3..495da1e1b 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ var CLI struct { TLSCertFile string `default:"" help:"File containing the default x509 Certificate for HTTPS."` TLSPrivateKeyFile string `default:"" help:"File containing the default x509 private key matching --tls-cert-file."` TLSClientCAFile string `default:"" help:"File containing the CA certificate for the client"` + OrgID string `default:"" help:"Multi tenant authorization"` } `cmd:"" help:"Runs Pyrra's API and UI."` Filesystem struct { ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use. Any non yaml files will be ignored."` @@ -84,6 +85,8 @@ var CLI struct { MimirBasicAuthUsername string `default:"" help:"The HTTP basic authentication username"` MimirBasicAuthPassword string `default:"" help:"The HTTP basic authentication password"` MimirWriteAlertingRules bool `default:"false" help:"If alerting rules should be provisioned to the Mimir Ruler."` + MimirOrgID string `default:"" help:"Multi tenant authorization"` + MimirDeploymentMode string `default:"monolithic" help:"Mimir deployment mode it can be monolithic or microservices mode"` } `cmd:"" help:"Runs Pyrra's Kubernetes operator and backend for the API."` Generate struct { ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."` @@ -128,7 +131,15 @@ func main() { if CLI.API.TLSClientCAFile != "" { clientConfig.TLSConfig = promconfig.TLSConfig{CAFile: CLI.API.TLSClientCAFile} } - + if CLI.API.OrgID != "" { + clientConfig.HTTPHeaders = &promconfig.Headers{ + Headers: map[string]promconfig.Header{ + "X-Scope-OrgID": { + Values: []string{CLI.API.OrgID}, + }, + }, + } + } roundTripper, err := promconfig.NewRoundTripperFromConfig(clientConfig, "prometheus") if err != nil { level.Error(logger).Log("msg", "failed to create API client round tripper", "err", err) @@ -162,6 +173,8 @@ func main() { PrometheusPrefix: CLI.Kubernetes.MimirPrometheusPrefix, BasicAuthUsername: CLI.Kubernetes.MimirBasicAuthUsername, BasicAuthPassword: CLI.Kubernetes.MimirBasicAuthPassword, + OrgID: CLI.Kubernetes.MimirOrgID, + DeploymentMode: CLI.Kubernetes.MimirDeploymentMode, } mimirClient, err = mimir.NewClient(mimirConfig) diff --git a/mimir/client.go b/mimir/client.go index 330ca008e..f0761c32c 100644 --- a/mimir/client.go +++ b/mimir/client.go @@ -13,6 +13,8 @@ type Client struct { client http.Client address *url.URL prometheusPrefix string + orgID string + deploymentMode string } // Config is used to configure the client. @@ -21,6 +23,8 @@ type Config struct { PrometheusPrefix string BasicAuthUsername string BasicAuthPassword string + OrgID string + DeploymentMode string } // NewClient creates a new client with the given configuration. @@ -48,6 +52,8 @@ func NewClient(config Config) (*Client, error) { client: httpClient, address: addr, prometheusPrefix: config.PrometheusPrefix, + orgID: config.OrgID, + deploymentMode: config.DeploymentMode, }, nil } @@ -66,13 +72,22 @@ func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error // Ready checks if mimir is ready to serve traffic. func (c *Client) Ready(ctx context.Context) error { - path := c.address.JoinPath("/ready") + path := c.address.JoinPath("/") + if c.deploymentMode == "monolithic" { + path = c.address.JoinPath("/ready") + } else if c.deploymentMode == "microservices" { + path = c.address.JoinPath("/api/v1/status/buildinfo") + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, path.String(), nil) if err != nil { return err } + if c.orgID != "" { + req.Header.Set("X-Scope-OrgID", c.orgID) + } + resp, err := c.client.Do(req) if err != nil { return err diff --git a/mimir/rulegroup.go b/mimir/rulegroup.go index 9684de737..cd02fbb87 100644 --- a/mimir/rulegroup.go +++ b/mimir/rulegroup.go @@ -27,6 +27,10 @@ func (c *Client) SetRuleGroup(ctx context.Context, namespace string, ruleGroup r req.Header.Set("Content-Type", "application/yaml") + if c.orgID != "" { + req.Header.Set("X-Scope-OrgID", c.orgID) + } + resp, err := c.client.Do(req) if err != nil { return err @@ -49,6 +53,10 @@ func (c *Client) DeleteNamespace(ctx context.Context, namespace string) error { return err } + if c.orgID != "" { + req.Header.Set("X-Scope-OrgID", c.orgID) + } + resp, err := c.client.Do(req) if err != nil { return err diff --git a/slo/rules.go b/slo/rules.go index ac469fac5..ed1b4b620 100644 --- a/slo/rules.go +++ b/slo/rules.go @@ -1170,6 +1170,7 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { return monitoringv1.RuleGroup{}, ErrGroupingUnsupported } + ruleLabels := o.commonRuleLabels(sloName) availability, err := parser.ParseExpr(`1 - sum(errorMetric{matchers="errors"} or vector(0)) / sum(metric{matchers="total"})`) if err != nil { return monitoringv1.RuleGroup{}, err @@ -1225,6 +1226,12 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { errorMatchers: errorMatchers, }.replace(availability) + for _, m := range totalMatchers { + if m.Type == labels.MatchEqual && m.Name != labels.MetricName { + ruleLabels[m.Name] = m.Value + } + } + rules = append(rules, monitoringv1.Rule{ Record: "pyrra_availability", Expr: intstr.FromString(availability.String()), @@ -1270,6 +1277,8 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { return monitoringv1.RuleGroup{}, ErrGroupingUnsupported } + ruleLabels := o.commonRuleLabels(sloName) + // availability { expr, err := parser.ParseExpr(`sum(errorMetric{matchers="errors"} or vector(0)) / sum(metric{matchers="total"})`) @@ -1306,6 +1315,12 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { Value: o.Name(), }) + for _, m := range errorMatchers { + if m.Type == labels.MatchEqual && m.Name != labels.MetricName { + ruleLabels[m.Name] = m.Value + } + } + objectiveReplacer{ metric: metric, matchers: matchers, @@ -1390,6 +1405,7 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { return monitoringv1.RuleGroup{}, ErrGroupingUnsupported } + ruleLabels := o.commonRuleLabels(sloName) totalMetric := countName(o.Indicator.BoolGauge.Metric.Name, o.Window) totalMatchers := cloneMatchers(o.Indicator.BoolGauge.Metric.LabelMatchers) for _, m := range totalMatchers { @@ -1418,6 +1434,11 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) { Value: o.Name(), }) + for _, m := range successMatchers { + if m.Type == labels.MatchEqual && m.Name != labels.MetricName { + ruleLabels[m.Name] = m.Value + } + } // availability { expr, err := parser.ParseExpr(`sum(errorMetric{matchers="errors"}) / sum(metric{matchers="total"})`) diff --git a/slo/rules_test.go b/slo/rules_test.go index 65d617e58..e2d17de51 100644 --- a/slo/rules_test.go +++ b/slo/rules_test.go @@ -1695,15 +1695,15 @@ func TestObjective_GrafanaRules(t *testing.T) { }, { Record: "pyrra_availability", Expr: intstr.FromString(`1 - sum(http_requests:increase4w{code=~"5..",job="thanos-receive-default",slo="monitoring-http-errors"} or vector(0)) / sum(http_requests:increase4w{job="thanos-receive-default",slo="monitoring-http-errors"})`), - Labels: map[string]string{"slo": "monitoring-http-errors"}, + Labels: map[string]string{"job": "thanos-receive-default", "slo": "monitoring-http-errors"}, }, { Record: "pyrra_requests_total", Expr: intstr.FromString(`sum(http_requests_total{job="thanos-receive-default"})`), - Labels: map[string]string{"slo": "monitoring-http-errors"}, + Labels: map[string]string{"job": "thanos-receive-default", "slo": "monitoring-http-errors"}, }, { Record: "pyrra_errors_total", Expr: intstr.FromString(`sum(http_requests_total{code=~"5..",job="thanos-receive-default"} or vector(0))`), - Labels: map[string]string{"slo": "monitoring-http-errors"}, + Labels: map[string]string{"job": "thanos-receive-default", "slo": "monitoring-http-errors"}, }}, }, }, { @@ -1731,15 +1731,15 @@ func TestObjective_GrafanaRules(t *testing.T) { }, { Record: "pyrra_availability", Expr: intstr.FromString(`1 - sum(grpc_server_handled:increase4w{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} or vector(0)) / sum(grpc_server_handled:increase4w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"})`), - Labels: map[string]string{"slo": "monitoring-grpc-errors"}, + Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors"}, }, { Record: "pyrra_requests_total", Expr: intstr.FromString(`sum(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"})`), - Labels: map[string]string{"slo": "monitoring-grpc-errors"}, + Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors"}, }, { Record: "pyrra_errors_total", Expr: intstr.FromString(`sum(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"} or vector(0))`), - Labels: map[string]string{"slo": "monitoring-grpc-errors"}, + Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors"}, }}, }, }, { @@ -1763,15 +1763,15 @@ func TestObjective_GrafanaRules(t *testing.T) { }, { Record: "pyrra_availability", Expr: intstr.FromString(`sum(http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="1",slo="monitoring-http-latency"} or vector(0)) / sum(http_request_duration_seconds:increase4w{code=~"2..",job="metrics-service-thanos-receive-default",le="",slo="monitoring-http-latency"})`), - Labels: map[string]string{"slo": "monitoring-http-latency"}, + Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "le": "1", "slo": "monitoring-http-latency"}, }, { Record: "pyrra_requests_total", Expr: intstr.FromString(`sum(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"})`), - Labels: map[string]string{"slo": "monitoring-http-latency"}, + Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "le": "1", "slo": "monitoring-http-latency"}, }, { Record: "pyrra_errors_total", Expr: intstr.FromString(`sum(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}) - sum(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"})`), - Labels: map[string]string{"slo": "monitoring-http-latency"}, + Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "le": "1", "slo": "monitoring-http-latency"}, }}, }, }, { @@ -1799,15 +1799,15 @@ func TestObjective_GrafanaRules(t *testing.T) { }, { Record: "pyrra_availability", Expr: intstr.FromString(`sum(grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6",slo="monitoring-grpc-latency"} or vector(0)) / sum(grpc_server_handling_seconds:increase1w{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="",slo="monitoring-grpc-latency"})`), - Labels: map[string]string{"slo": "monitoring-grpc-latency"}, + Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "le": "0.6", "slo": "monitoring-grpc-latency"}, }, { Record: "pyrra_requests_total", Expr: intstr.FromString(`sum(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"})`), - Labels: map[string]string{"slo": "monitoring-grpc-latency"}, + Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "le": "0.6", "slo": "monitoring-grpc-latency"}, }, { Record: "pyrra_errors_total", Expr: intstr.FromString(`sum(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}) - sum(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"})`), - Labels: map[string]string{"slo": "monitoring-grpc-latency"}, + Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "le": "0.6", "slo": "monitoring-grpc-latency"}, }}, }, }, { @@ -1863,15 +1863,15 @@ func TestObjective_GrafanaRules(t *testing.T) { }, { Record: "pyrra_availability", Expr: intstr.FromString(`1 - sum(apiserver_request:increase2w{code=~"5..",job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} or vector(0)) / sum(apiserver_request:increase2w{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"})`), - Labels: map[string]string{"slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "pyrra_requests_total", Expr: intstr.FromString(`sum(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"})`), - Labels: map[string]string{"slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { Record: "pyrra_errors_total", Expr: intstr.FromString(`sum(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"} or vector(0))`), - Labels: map[string]string{"slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }}, }, }, { diff --git a/slo/slo.go b/slo/slo.go index 6cc2a0eb5..d0fab7be4 100644 --- a/slo/slo.go +++ b/slo/slo.go @@ -17,15 +17,15 @@ const ( ) type Objective struct { - Labels labels.Labels - Annotations map[string]string - Description string - Target float64 - Window model.Duration - Config string - - Alerting Alerting - Indicator Indicator + Labels labels.Labels + Annotations map[string]string + Description string + Target float64 + Window model.Duration + Config string + RecordingRuleNamespace string + Alerting Alerting + Indicator Indicator } func (o Objective) Name() string {