diff --git a/filesystem.go b/filesystem.go index b6a743db3..34a98e130 100644 --- a/filesystem.go +++ b/filesystem.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "os" "path/filepath" "sync" @@ -89,7 +90,7 @@ Objectives: return objectives } -func cmdFilesystem(logger log.Logger, reg *prometheus.Registry, promClient api.Client, configFiles, prometheusFolder string, genericRules bool) int { +func cmdFilesystem(logger log.Logger, reg *prometheus.Registry, promClient api.Client, configFiles, prometheusFolder string, genericRules bool, pyrraExternalURL *url.URL) int { reconcilesTotal := prometheus.NewCounter(prometheus.CounterOpts{ Name: "pyrra_filesystem_reconciles_total", Help: "The total amount of reconciles.", @@ -179,7 +180,12 @@ func cmdFilesystem(logger log.Logger, reg *prometheus.Registry, promClient api.C level.Debug(logger).Log("msg", "processing", "file", f) reconcilesTotal.Inc() - err := writeRuleFile(logger, f, prometheusFolder, genericRules, false) + pyrraURL := "" + if pyrraExternalURL != nil { + pyrraURL = pyrraExternalURL.String() + } + + err := writeRuleFile(logger, f, prometheusFolder, genericRules, false, pyrraURL) if err != nil { reconcilesErrors.Inc() level.Error(logger).Log("msg", "error creating rule file", "file", f, "err", err) @@ -301,7 +307,7 @@ func (s *FilesystemObjectiveServer) List(_ context.Context, req *connect.Request }), nil } -func writeRuleFile(logger log.Logger, file, prometheusFolder string, genericRules, operatorRule bool) error { +func writeRuleFile(logger log.Logger, file, prometheusFolder string, genericRules, operatorRule bool, externalURL string) error { kubeObjective, objective, err := objectiveFromFile(file) if err != nil { return fmt.Errorf("failed to get objective: %w", err) @@ -326,7 +332,7 @@ func writeRuleFile(logger log.Logger, file, prometheusFolder string, genericRule return fmt.Errorf("failed to get increase rules: %w", err) } - burnrates, err := objective.Burnrates() + burnrates, err := objective.Burnrates(externalURL) if err != nil { return fmt.Errorf("failed to get burn rate rules: %w", err) } diff --git a/generate.go b/generate.go index 77e975f66..b08f4b39f 100644 --- a/generate.go +++ b/generate.go @@ -14,21 +14,27 @@ limitations under the License. package main import ( + "net/url" "path/filepath" "github.com/go-kit/log" "github.com/go-kit/log/level" ) -func cmdGenerate(logger log.Logger, configFiles, prometheusFolder string, genericRules, operatorRule bool) int { +func cmdGenerate(logger log.Logger, configFiles, prometheusFolder string, genericRules, operatorRule bool, externalURL *url.URL) int { filenames, err := filepath.Glob(configFiles) if err != nil { level.Error(logger).Log("msg", "getting file names", "err", err) return 1 } + externalURLStr := "" + if externalURL != nil { + externalURLStr = externalURL.String() + } + for _, file := range filenames { - err := writeRuleFile(logger, file, prometheusFolder, genericRules, operatorRule) + err := writeRuleFile(logger, file, prometheusFolder, genericRules, operatorRule, externalURLStr) if err != nil { level.Error(logger).Log("msg", "generating rule files", "err", err) return 1 diff --git a/kubernetes.go b/kubernetes.go index 10033e4c6..9bbbf65f5 100644 --- a/kubernetes.go +++ b/kubernetes.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net/http" + "net/url" "os" "syscall" "time" @@ -66,6 +67,7 @@ func cmdKubernetes( certFile, privateKeyFile string, mimirClient *mimir.Client, mimirWriteAlertingRules bool, + pyrraExternalURL *url.URL, ) int { setupLog := ctrl.Log.WithName("setup") ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -86,6 +88,11 @@ func cmdKubernetes( os.Exit(1) } + pyrraURL := "" + if pyrraExternalURL != nil { + pyrraURL = pyrraExternalURL.String() + } + reconciler := &controllers.ServiceLevelObjectiveReconciler{ Client: mgr.GetClient(), Logger: log.With(logger, "controllers", "ServiceLevelObjective"), @@ -93,6 +100,7 @@ func cmdKubernetes( ConfigMapMode: configMapMode, MimirClient: mimirClient, MimirWriteAlertingRules: mimirWriteAlertingRules, + PyrraExternalURL: pyrraURL, } if err = reconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ServiceLevelObjective") @@ -129,7 +137,8 @@ func cmdKubernetes( { router := http.NewServeMux() router.Handle(objectivesv1alpha1connect.NewObjectiveBackendServiceHandler(&KubernetesObjectiveServer{ - client: mgr.GetClient(), + client: mgr.GetClient(), + pyrraURL: pyrraURL, })) server := http.Server{ @@ -166,7 +175,8 @@ type KubernetesClient interface { } type KubernetesObjectiveServer struct { - client KubernetesClient + client KubernetesClient + pyrraURL string } func (s *KubernetesObjectiveServer) List(ctx context.Context, req *connect.Request[objectivesv1alpha1.ListRequest]) (*connect.Response[objectivesv1alpha1.ListResponse], error) { @@ -206,19 +216,19 @@ func (s *KubernetesObjectiveServer) List(ctx context.Context, req *connect.Reque } objectives := make([]*objectivesv1alpha1.Objective, 0, len(list.Items)) - for _, s := range list.Items { + for _, slo := range list.Items { if nameMatcher != nil { - if !nameMatcher.Matches(s.GetName()) { + if !nameMatcher.Matches(slo.GetName()) { continue } } if namespaceMatcher != nil { - if !namespaceMatcher.Matches(s.GetNamespace()) { + if !namespaceMatcher.Matches(slo.GetNamespace()) { continue } } - internal, err := s.Internal() + internal, err := slo.Internal() if err != nil { return nil, connect.NewError(connect.CodeInternal, err) } diff --git a/kubernetes/controllers/servicelevelobjective.go b/kubernetes/controllers/servicelevelobjective.go index 6e768b30b..b7a17255c 100644 --- a/kubernetes/controllers/servicelevelobjective.go +++ b/kubernetes/controllers/servicelevelobjective.go @@ -48,6 +48,7 @@ type ServiceLevelObjectiveReconciler struct { MimirWriteAlertingRules bool Logger kitlog.Logger Scheme *runtime.Scheme + PyrraExternalURL string ConfigMapMode bool GenericRules bool } @@ -106,7 +107,7 @@ func (r *ServiceLevelObjectiveReconciler) Reconcile(ctx context.Context, req ctr } func (r *ServiceLevelObjectiveReconciler) reconcilePrometheusRule(ctx context.Context, logger kitlog.Logger, req ctrl.Request, kubeObjective pyrrav1alpha1.ServiceLevelObjective) (ctrl.Result, error) { - newRule, err := makePrometheusRule(kubeObjective, r.GenericRules) + newRule, err := makePrometheusRule(kubeObjective, r.GenericRules, r.PyrraExternalURL) if err != nil { return ctrl.Result{}, err } @@ -139,7 +140,7 @@ func (r *ServiceLevelObjectiveReconciler) reconcilePrometheusRule(ctx context.Co } func (r *ServiceLevelObjectiveReconciler) reconcileMimirRuleGroup(ctx context.Context, logger kitlog.Logger, kubeObjective pyrrav1alpha1.ServiceLevelObjective) (ctrl.Result, error) { - newRuleGroup, err := makeMimirRuleGroup(kubeObjective, r.GenericRules, r.MimirWriteAlertingRules) + newRuleGroup, err := makeMimirRuleGroup(kubeObjective, r.GenericRules, r.MimirWriteAlertingRules, r.PyrraExternalURL) if err != nil { return ctrl.Result{}, err } @@ -171,7 +172,7 @@ func (r *ServiceLevelObjectiveReconciler) reconcileConfigMap( ) (ctrl.Result, error) { name := fmt.Sprintf("pyrra-recording-rule-%s", kubeObjective.GetName()) - newConfigMap, err := makeConfigMap(name, kubeObjective, r.GenericRules) + newConfigMap, err := makeConfigMap(name, kubeObjective, r.GenericRules, r.PyrraExternalURL) if err != nil { return ctrl.Result{}, err } @@ -218,7 +219,7 @@ func (r *ServiceLevelObjectiveReconciler) SetupWebhookWithManager(mgr ctrl.Manag Complete() } -func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool) (*corev1.ConfigMap, error) { +func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool, externalURL string) (*corev1.ConfigMap, error) { objective, err := kubeObjective.Internal() if err != nil { return nil, fmt.Errorf("failed to get objective: %w", err) @@ -228,7 +229,7 @@ func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjectiv if err != nil { return nil, fmt.Errorf("failed to get increase rules: %w", err) } - burnrates, err := objective.Burnrates() + burnrates, err := objective.Burnrates(externalURL) if err != nil { return nil, fmt.Errorf("failed to get burn rate rules: %w", err) } @@ -286,7 +287,7 @@ func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjectiv }, nil } -func makeMimirRuleGroup(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules, writeAlertingRules bool) (*rulefmt.RuleGroup, error) { +func makeMimirRuleGroup(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules, writeAlertingRules bool, externalURL string) (*rulefmt.RuleGroup, error) { objective, err := kubeObjective.Internal() if err != nil { return nil, fmt.Errorf("failed to get objective: %w", err) @@ -298,7 +299,7 @@ func makeMimirRuleGroup(kubeObjective pyrrav1alpha1.ServiceLevelObjective, gener } increasesMimirRules := prometheusRulesToMimirRules(increases.Rules, writeAlertingRules) - burnrates, err := objective.Burnrates() + burnrates, err := objective.Burnrates(externalURL) if err != nil { return nil, fmt.Errorf("failed to get burn rate rules: %w", err) } @@ -372,7 +373,7 @@ func prometheusRulesToMimirRules(promRules []monitoringv1.Rule, writeAlertingRul return rules } -func makePrometheusRule(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool) (*monitoringv1.PrometheusRule, error) { +func makePrometheusRule(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool, externalURL string) (*monitoringv1.PrometheusRule, error) { objective, err := kubeObjective.Internal() if err != nil { return nil, fmt.Errorf("failed to get objective: %w", err) @@ -382,7 +383,7 @@ func makePrometheusRule(kubeObjective pyrrav1alpha1.ServiceLevelObjective, gener if err != nil { return nil, fmt.Errorf("failed to get increase rules: %w", err) } - burnrates, err := objective.Burnrates() + burnrates, err := objective.Burnrates(externalURL) if err != nil { return nil, fmt.Errorf("failed to get burn rate rules: %w", err) } diff --git a/main.go b/main.go index e7dd6f200..350eb193f 100644 --- a/main.go +++ b/main.go @@ -75,6 +75,7 @@ var CLI struct { PrometheusURL *url.URL `default:"http://localhost:9090" help:"The URL to the Prometheus to query."` PrometheusFolder string `default:"/etc/prometheus/pyrra/" help:"The folder where Pyrra writes the generates Prometheus rules and alerts."` GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."` + ExternalURL *url.URL `default:"http://localhost:9099" help:"The URL for Pyrra to be included in alert annotations. This will be used to generate direct links to the Pyrra UI in alerts."` } `cmd:"" help:"Runs Pyrra's filesystem operator and backend for the API."` Kubernetes struct { MetricsAddr string `default:":8080" help:"The address the metric endpoint binds to."` @@ -88,12 +89,14 @@ var CLI struct { MimirBasicAuthUsername string `default:"" help:"The HTTP basic authentication username"` MimirBasicAuthPassword string `default:"" help:"The HTTP basic authentication password"` MimirWriteAlertingRules bool `default:"false" help:"If alerting rules should be provisioned to the Mimir Ruler."` + ExternalURL *url.URL `default:"http://localhost:9099" help:"The URL for Pyrra to be included in alert annotations. This will be used to generate direct links to the Pyrra UI in alerts."` } `cmd:"" help:"Runs Pyrra's Kubernetes operator and backend for the API."` Generate struct { - ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."` - PrometheusFolder string `default:"/etc/prometheus/pyrra/" help:"The folder where Pyrra writes the generated Prometheus rules and alerts."` - GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."` - OperatorRule bool `default:"false" help:"Generate rule files as prometheus-operator PrometheusRule: https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PrometheusRule."` + ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."` + PrometheusFolder string `default:"/etc/prometheus/pyrra/" help:"The folder where Pyrra writes the generated Prometheus rules and alerts."` + GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."` + OperatorRule bool `default:"false" help:"Generate rule files as prometheus-operator PrometheusRule: https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PrometheusRule."` + ExternalURL *url.URL `default:"http://localhost:9099" help:"The URL for Pyrra to be included in alert annotations. This will be used to generate direct links to the Pyrra UI in alerts."` } `cmd:"" help:"Read SLO config files and rewrites them as Prometheus rules and alerts."` } @@ -162,7 +165,7 @@ func main() { level.Info(logger).Log("msg", "using Prometheus", "url", prometheusURL.String()) // Default external url to prometheus if not defined - externalURL := prometheusURL + externalDatasourceURL := prometheusURL if CLI.API.PrometheusExternalURL != nil && CLI.API.GrafanaExternalURL != nil { level.Error(logger).Log("msg", "prometheus external URL set alongside grafana external url") os.Exit(1) @@ -170,13 +173,13 @@ func main() { level.Error(logger).Log("msg", "grafana external datasource id set without grafana external url") os.Exit(1) } else if CLI.API.PrometheusExternalURL != nil { - externalURL = CLI.API.PrometheusExternalURL + externalDatasourceURL = CLI.API.PrometheusExternalURL } else if CLI.API.GrafanaExternalURL != nil { if CLI.API.GrafanaExternalDatasourceID == "" { level.Error(logger).Log("msg", "grafana external datasource id cannot be empty when using an external grafana url") os.Exit(1) } - externalURL = CLI.API.GrafanaExternalURL + externalDatasourceURL = CLI.API.GrafanaExternalURL } // Mimir Client @@ -211,7 +214,7 @@ func main() { logger, reg, client, - externalURL, + externalDatasourceURL, CLI.API.APIURL, CLI.API.GrafanaExternalOrgID, CLI.API.GrafanaExternalDatasourceID, @@ -228,6 +231,7 @@ func main() { CLI.Filesystem.ConfigFiles, CLI.Filesystem.PrometheusFolder, CLI.Filesystem.GenericRules, + CLI.Filesystem.ExternalURL, ) case "kubernetes": code = cmdKubernetes( @@ -240,6 +244,7 @@ func main() { CLI.Kubernetes.TLSPrivateKeyFile, mimirClient, CLI.Kubernetes.MimirWriteAlertingRules, + CLI.Kubernetes.ExternalURL, ) case "generate": code = cmdGenerate( @@ -248,6 +253,7 @@ func main() { CLI.Generate.PrometheusFolder, CLI.Generate.GenericRules, CLI.Generate.OperatorRule, + CLI.Generate.ExternalURL, ) } os.Exit(code) @@ -257,7 +263,7 @@ func cmdAPI( logger log.Logger, reg *prometheus.Registry, promClient api.Client, - externalURL, apiURL *url.URL, + externalDatasourceURL, apiURL *url.URL, externalGrafanaOrgID, externalGrafanaDatasourceID string, routePrefix, uiRoutePrefix string, tlsCertFile, tlsPrivateKeyFile string, @@ -277,9 +283,9 @@ func cmdAPI( } if externalGrafanaDatasourceID == "" { - level.Info(logger).Log("msg", "UI redirect to Prometheus", "url", externalURL.String()) + level.Info(logger).Log("msg", "UI redirect to Prometheus", "url", externalDatasourceURL.String()) } else { - level.Info(logger).Log("msg", "UI redirect to Grafana", "url", externalURL.String(), + level.Info(logger).Log("msg", "UI redirect to Grafana", "url", externalDatasourceURL.String(), "datasourceId", externalGrafanaDatasourceID, "orgId", externalGrafanaOrgID) } level.Info(logger).Log("msg", "using API at", "url", apiURL.String()) @@ -370,13 +376,13 @@ func cmdAPI( r.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) r.Get("/objectives", func(w http.ResponseWriter, _ *http.Request) { err := tmpl.Execute(w, struct { - ExternalURL string + ExternalDatasourceURL string ExternalGrafanaDatasourceID string ExternalGrafanaOrgID string PathPrefix string APIBasepath string }{ - ExternalURL: externalURL.String(), + ExternalDatasourceURL: externalDatasourceURL.String(), ExternalGrafanaDatasourceID: externalGrafanaDatasourceID, ExternalGrafanaOrgID: externalGrafanaOrgID, PathPrefix: uiRoutePrefix, @@ -396,7 +402,7 @@ func cmdAPI( PathPrefix string APIBasepath string }{ - ExternalURL: externalURL.String(), + ExternalURL: externalDatasourceURL.String(), ExternalGrafanaDatasourceID: externalGrafanaDatasourceID, ExternalGrafanaOrgID: externalGrafanaOrgID, PathPrefix: uiRoutePrefix, diff --git a/slo/rules.go b/slo/rules.go index 588bc445a..1f656c1c4 100644 --- a/slo/rules.go +++ b/slo/rules.go @@ -3,6 +3,7 @@ package slo import ( "errors" "fmt" + "net/url" "sort" "strconv" "strings" @@ -56,7 +57,7 @@ func (o Objective) Alerts() ([]MultiBurnRateAlert, error) { return mbras, nil } -func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { +func (o Objective) Burnrates(externalURL string) (monitoringv1.RuleGroup, error) { sloName := o.Labels.Get(labels.MetricName) ws := Windows(time.Duration(o.Window)) @@ -118,7 +119,7 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { for _, w := range ws { alertLabels := o.commonRuleLabels(sloName) - alertAnnotations := o.commonRuleAnnotations() + alertAnnotations := o.commonRuleAnnotations(externalURL) for _, m := range matchers { if m.Type == labels.MatchEqual && m.Name != labels.MetricName { if _, ok := groupingMap[m.Name]; !ok { // only add labels that aren't grouped by @@ -206,7 +207,7 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { for _, w := range ws { alertLabels := o.commonRuleLabels(sloName) - alertAnnotations := o.commonRuleAnnotations() + alertAnnotations := o.commonRuleAnnotations(externalURL) for _, m := range matchers { if m.Type == labels.MatchEqual && m.Name != labels.MetricName { if _, ok := groupingMap[m.Name]; !ok { // only add labels that aren't grouped by @@ -294,7 +295,7 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { for _, w := range ws { alertLabels := o.commonRuleLabels(sloName) - alertAnnotations := o.commonRuleAnnotations() + alertAnnotations := o.commonRuleAnnotations(externalURL) for _, m := range matchers { if m.Type == labels.MatchEqual && m.Name != labels.MetricName { if _, ok := groupingMap[m.Name]; !ok { // only add labels that aren't grouped by @@ -382,7 +383,7 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { for _, w := range ws { alertLabels := o.commonRuleLabels(sloName) - alertAnnotations := o.commonRuleAnnotations() + alertAnnotations := o.commonRuleAnnotations(externalURL) for _, m := range matchers { if m.Type == labels.MatchEqual && m.Name != labels.MetricName { if _, ok := groupingMap[m.Name]; !ok { // only add labels that aren't grouped by @@ -605,13 +606,48 @@ func (o Objective) commonRuleLabels(sloName string) map[string]string { return ruleLabels } -func (o Objective) commonRuleAnnotations() map[string]string { +func (o Objective) commonRuleAnnotations(externalURL string) map[string]string { var annotations map[string]string - if len(o.Annotations) > 0 { - annotations = make(map[string]string) - for key, value := range o.Annotations { - if strings.HasPrefix(key, PropagationLabelsPrefix) { - annotations[strings.TrimPrefix(key, PropagationLabelsPrefix)] = value + + // Add existing annotations from the objective + for key, value := range o.Annotations { + if strings.HasPrefix(key, PropagationLabelsPrefix) { + if annotations == nil { + annotations = make(map[string]string) + } + annotations[strings.TrimPrefix(key, PropagationLabelsPrefix)] = value + } + } + + // Add External URL if provided + if externalURL != "" { + sloName := o.Labels.Get(labels.MetricName) + if sloName != "" { + if annotations == nil { + annotations = make(map[string]string) + } + + externalURLParsed, err := url.Parse(externalURL) + if err != nil { + return nil + } + + params := url.Values{} + params.Add("expr", `{__name__="`+sloName+`"}`) + params.Add("from", "now-1h") + params.Add("to", "now") + + externalURLParsed.Path = "/objectives" + externalURLParsed.RawQuery = params.Encode() + + annotations["pyrra_url"] = externalURLParsed.String() + + if grouping := o.Grouping(); len(grouping) > 0 { + groups := make([]string, 0, len(grouping)) + for _, l := range grouping { + groups = append(groups, l+"=%22{{$labels."+l+"}}%22") + } + annotations["pyrra_url"] += "&grouping=%7B" + strings.Join(groups, ",") + "%7D" } } } @@ -717,7 +753,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { Expr: intstr.FromString(expr.String()), For: monitoringDuration(o.AbsentDuration().String()), Labels: alertLabels, - Annotations: o.commonRuleAnnotations(), + Annotations: o.commonRuleAnnotations(""), }) } @@ -757,7 +793,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { Expr: intstr.FromString(expr.String()), For: monitoringDuration(o.AbsentDuration().String()), Labels: alertLabels, - Annotations: o.commonRuleAnnotations(), + Annotations: o.commonRuleAnnotations(""), }) } } @@ -867,7 +903,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { Expr: intstr.FromString(expr.String()), For: monitoringDuration(o.AbsentDuration().String()), Labels: alertLabels, - Annotations: o.commonRuleAnnotations(), + Annotations: o.commonRuleAnnotations(""), }) expr, err = absentExpr() @@ -892,7 +928,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { Expr: intstr.FromString(expr.String()), For: monitoringDuration(o.AbsentDuration().String()), Labels: alertLabelsLe, - Annotations: o.commonRuleAnnotations(), + Annotations: o.commonRuleAnnotations(""), }) } case LatencyNative: @@ -1033,7 +1069,7 @@ func (o Objective) IncreaseRules() (monitoringv1.RuleGroup, error) { Expr: intstr.FromString(expr.String()), For: monitoringDuration(o.AbsentDuration().String()), Labels: alertLabels, - Annotations: o.commonRuleAnnotations(), + Annotations: o.commonRuleAnnotations(""), }) } } diff --git a/slo/rules_test.go b/slo/rules_test.go index fdfa494c1..4736c6674 100644 --- a/slo/rules_test.go +++ b/slo/rules_test.go @@ -50,25 +50,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum(rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[4d])) / sum(rate(http_requests_total{job="thanos-receive-default"}[4d]))`), Labels: map[string]string{"job": "thanos-receive-default", "slo": "monitoring-http-errors"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m0s"), - Expr: intstr.FromString(`http_requests:burnrate5m{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "1h", "slo": "monitoring-http-errors", "short": "5m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m0s"), - Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h0m0s"), - Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h0m0s"), - Expr: intstr.FromString(`http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m0s"), + Expr: intstr.FromString(`http_requests:burnrate5m{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "1h", "slo": "monitoring-http-errors", "short": "5m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m0s"), + Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h0m0s"), + Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h0m0s"), + Expr: intstr.FromString(`http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -106,25 +110,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",job="thanos-receive-default"}[4d])) / sum by (handler, job) (rate(http_requests_total{job="thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m0s"), - Expr: intstr.FromString(`http_requests:burnrate5m{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-errors", "short": "5m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m0s"), - Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h0m0s"), - Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h0m0s"), - Expr: intstr.FromString(`http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m0s"), + Expr: intstr.FromString(`http_requests:burnrate5m{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-errors", "short": "5m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m0s"), + Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h0m0s"), + Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h0m0s"), + Expr: intstr.FromString(`http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, }}, }, }, { @@ -162,25 +170,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum by (handler, job) (rate(http_requests_total{code=~"5..",handler=~"/api.*",job="thanos-receive-default"}[4d])) / sum by (handler, job) (rate(http_requests_total{handler=~"/api.*",job="thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-errors"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m0s"), - Expr: intstr.FromString(`http_requests:burnrate5m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-errors", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m0s"), - Expr: intstr.FromString(`http_requests:burnrate30m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h0m0s"), - Expr: intstr.FromString(`http_requests:burnrate2h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h0m0s"), - Expr: intstr.FromString(`http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m0s"), + Expr: intstr.FromString(`http_requests:burnrate5m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-errors", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m0s"), + Expr: intstr.FromString(`http_requests:burnrate30m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h0m0s"), + Expr: intstr.FromString(`http_requests:burnrate2h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h0m0s"), + Expr: intstr.FromString(`http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, }}, }, }, { @@ -218,25 +230,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum(rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d])) / sum(rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors"}, }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate5m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999)) and grpc_server_handled:burnrate1h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999))`), - For: monitoringDuration("2m0s"), - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999))`), - For: monitoringDuration("15m0s"), - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999))`), - For: monitoringDuration("1h0m0s"), - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999)) and grpc_server_handled:burnrate4d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999))`), - For: monitoringDuration("3h0m0s"), - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate5m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999)) and grpc_server_handled:burnrate1h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999))`), + For: monitoringDuration("2m0s"), + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999))`), + For: monitoringDuration("15m0s"), + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999))`), + For: monitoringDuration("1h0m0s"), + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999)) and grpc_server_handled:burnrate4d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999))`), + For: monitoringDuration("3h0m0s"), + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -274,25 +290,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum by (handler, job) (rate(grpc_server_handled_total{grpc_code=~"Aborted|Unavailable|Internal|Unknown|Unimplemented|DataLoss",grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d])) / sum by (handler, job) (rate(grpc_server_handled_total{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[4d]))`), Labels: map[string]string{"grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors"}, }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate5m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999)) and grpc_server_handled:burnrate1h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999))`), - For: monitoringDuration("2m0s"), - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999))`), - For: monitoringDuration("15m0s"), - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999))`), - For: monitoringDuration("1h0m0s"), - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999)) and grpc_server_handled:burnrate4d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999))`), - For: monitoringDuration("3h0m0s"), - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate5m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999)) and grpc_server_handled:burnrate1h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999))`), + For: monitoringDuration("2m0s"), + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999))`), + For: monitoringDuration("15m0s"), + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999))`), + For: monitoringDuration("1h0m0s"), + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999)) and grpc_server_handled:burnrate4d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999))`), + For: monitoringDuration("3h0m0s"), + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-errors%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, }}, }, }, { @@ -330,25 +350,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum(rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d])) - sum(rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum(rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d]))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -386,25 +410,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`1 - histogram_fraction(0, 1, sum(rate(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}[4d])))`), Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -442,25 +470,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",job="metrics-service-thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h"), - Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h"), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, }}, }, }, { @@ -498,25 +530,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4d])) - sum by (handler, job) (rate(http_request_duration_seconds_bucket{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default",le="1"}[4d]))) / sum by (handler, job) (rate(http_request_duration_seconds_count{code=~"2..",handler=~"/api.*",job="metrics-service-thanos-receive-default"}[4d]))`), Labels: map[string]string{"slo": "monitoring-http-latency"}, }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - For: monitoringDuration("2m"), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-latency", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - For: monitoringDuration("15m"), - Labels: map[string]string{"severity": "critical", "long": "6h", "short": "30m", "slo": "monitoring-http-latency", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - For: monitoringDuration("1h"), - Labels: map[string]string{"severity": "warning", "long": "1d", "short": "2h", "slo": "monitoring-http-latency", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - For: monitoringDuration("3h"), - Labels: map[string]string{"severity": "warning", "long": "4d", "short": "6h", "slo": "monitoring-http-latency", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), + For: monitoringDuration("2m"), + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-latency", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), + For: monitoringDuration("15m"), + Labels: map[string]string{"severity": "critical", "long": "6h", "short": "30m", "slo": "monitoring-http-latency", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), + For: monitoringDuration("1h"), + Labels: map[string]string{"severity": "warning", "long": "1d", "short": "2h", "slo": "monitoring-http-latency", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), + For: monitoringDuration("3h"), + Labels: map[string]string{"severity": "warning", "long": "4d", "short": "6h", "slo": "monitoring-http-latency", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-http-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, }}, }, }, { @@ -554,25 +590,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum(rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d])) - sum(rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1d]))) / sum(rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995)) and grpc_server_handling_seconds:burnrate15m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995))`), - For: monitoringDuration("1m"), - Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "12h"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995))`), - For: monitoringDuration("4m"), - Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995))`), - For: monitoringDuration("15m"), - Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "3d12h"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995)) and grpc_server_handling_seconds:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995))`), - For: monitoringDuration("45m"), - Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1w"}, + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995)) and grpc_server_handling_seconds:burnrate15m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995))`), + For: monitoringDuration("1m"), + Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "12h"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995))`), + For: monitoringDuration("4m"), + Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995))`), + For: monitoringDuration("15m"), + Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "3d12h"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995)) and grpc_server_handling_seconds:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995))`), + For: monitoringDuration("45m"), + Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -610,25 +650,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d])) - sum by (handler, job) (rate(grpc_server_handling_seconds_bucket{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",le="0.6"}[1d]))) / sum by (handler, job) (rate(grpc_server_handling_seconds_count{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api"}[1d]))`), Labels: map[string]string{"slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995)) and grpc_server_handling_seconds:burnrate15m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995))`), - For: monitoringDuration("1m"), - Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "12h"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995))`), - For: monitoringDuration("4m"), - Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995))`), - For: monitoringDuration("15m"), - Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "3d12h"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995)) and grpc_server_handling_seconds:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995))`), - For: monitoringDuration("45m"), - Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1w"}, + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995)) and grpc_server_handling_seconds:burnrate15m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995))`), + For: monitoringDuration("1m"), + Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "12h"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995))`), + For: monitoringDuration("4m"), + Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995))`), + For: monitoringDuration("15m"), + Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "3d12h"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995)) and grpc_server_handling_seconds:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995))`), + For: monitoringDuration("45m"), + Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-grpc-latency%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,handler=%22{{$labels.handler}}%22%7D"}, }}, }, }, { @@ -666,25 +710,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum(rate(prometheus_operator_reconcile_errors_total[2d])) / sum(rate(prometheus_operator_reconcile_operations_total[2d]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate30m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m", "exhaustion": "1d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("8m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("30m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h", "exhaustion": "1w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h30m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate2d{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h", "exhaustion": "2w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate30m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("8m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("30m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h30m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate2d{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -722,25 +770,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum by (namespace) (rate(prometheus_operator_reconcile_errors_total[2d])) / sum by (namespace) (rate(prometheus_operator_reconcile_operations_total[2d]))`), Labels: map[string]string{"slo": "monitoring-prometheus-operator-errors"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate30m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m", "exhaustion": "1d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("8m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("30m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h", "exhaustion": "1w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h30m0s"), - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate2d{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h", "exhaustion": "2w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate30m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now&grouping=%7Bnamespace=%22{{$labels.namespace}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("8m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now&grouping=%7Bnamespace=%22{{$labels.namespace}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("30m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now&grouping=%7Bnamespace=%22{{$labels.namespace}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h30m0s"), + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate2d{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22monitoring-prometheus-operator-errors%22%7D&from=now-1h&to=now&grouping=%7Bnamespace=%22{{$labels.namespace}}%22%7D"}, }}, }, }, { @@ -778,25 +830,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum(rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d])) / sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate3m{job="apiserver",slo="apiserver-write-response-errors"} > (14 * (1-0.99)) and apiserver_request:burnrate30m{job="apiserver",slo="apiserver-write-response-errors"} > (14 * (1-0.99))`), - For: monitoringDuration("1m0s"), - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1d"}, - }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate15m{job="apiserver",slo="apiserver-write-response-errors"} > (7 * (1-0.99)) and apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors"} > (7 * (1-0.99))`), - For: monitoringDuration("8m0s"), - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2d"}, - }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate1h{job="apiserver",slo="apiserver-write-response-errors"} > (2 * (1-0.99)) and apiserver_request:burnrate12h{job="apiserver",slo="apiserver-write-response-errors"} > (2 * (1-0.99))`), - For: monitoringDuration("30m0s"), - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1w"}, - }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors"} > (1 * (1-0.99)) and apiserver_request:burnrate2d{job="apiserver",slo="apiserver-write-response-errors"} > (1 * (1-0.99))`), - For: monitoringDuration("1h30m0s"), - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2w"}, + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate3m{job="apiserver",slo="apiserver-write-response-errors"} > (14 * (1-0.99)) and apiserver_request:burnrate30m{job="apiserver",slo="apiserver-write-response-errors"} > (14 * (1-0.99))`), + For: monitoringDuration("1m0s"), + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate15m{job="apiserver",slo="apiserver-write-response-errors"} > (7 * (1-0.99)) and apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors"} > (7 * (1-0.99))`), + For: monitoringDuration("8m0s"), + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate1h{job="apiserver",slo="apiserver-write-response-errors"} > (2 * (1-0.99)) and apiserver_request:burnrate12h{job="apiserver",slo="apiserver-write-response-errors"} > (2 * (1-0.99))`), + For: monitoringDuration("30m0s"), + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now"}, + }, { + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors"} > (1 * (1-0.99)) and apiserver_request:burnrate2d{job="apiserver",slo="apiserver-write-response-errors"} > (1 * (1-0.99))`), + For: monitoringDuration("1h30m0s"), + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -834,25 +890,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`sum by (verb) (rate(apiserver_request_total{code=~"5..",job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d])) / sum by (verb) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-write-response-errors"}, }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate3m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (14 * (1-0.99)) and apiserver_request:burnrate30m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (14 * (1-0.99))`), - For: monitoringDuration("1m0s"), - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1d"}, - }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate15m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (7 * (1-0.99)) and apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (7 * (1-0.99))`), - For: monitoringDuration("8m0s"), - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2d"}, - }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate1h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2 * (1-0.99)) and apiserver_request:burnrate12h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2 * (1-0.99))`), - For: monitoringDuration("30m0s"), - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1w"}, - }, { - Alert: "APIServerErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (1 * (1-0.99)) and apiserver_request:burnrate2d{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (1 * (1-0.99))`), - For: monitoringDuration("1h30m0s"), - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2w"}, + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate3m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (14 * (1-0.99)) and apiserver_request:burnrate30m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (14 * (1-0.99))`), + For: monitoringDuration("1m0s"), + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now&grouping=%7Bverb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate15m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (7 * (1-0.99)) and apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (7 * (1-0.99))`), + For: monitoringDuration("8m0s"), + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now&grouping=%7Bverb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate1h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2 * (1-0.99)) and apiserver_request:burnrate12h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2 * (1-0.99))`), + For: monitoringDuration("30m0s"), + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now&grouping=%7Bverb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "APIServerErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (1 * (1-0.99)) and apiserver_request:burnrate2d{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (1 * (1-0.99))`), + For: monitoringDuration("1h30m0s"), + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-write-response-errors%22%7D&from=now-1h&to=now&grouping=%7Bverb=%22{{$labels.verb}}%22%7D"}, }}, }, }, { @@ -926,25 +986,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99)) and apiserver_request_duration_seconds:burnrate30m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99))`), - For: monitoringDuration("1m"), - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99))`), - For: monitoringDuration("8m"), - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99))`), - For: monitoringDuration("30m"), - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1w"}, - }, { - Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99)) and apiserver_request_duration_seconds:burnrate2d{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99))`), - For: monitoringDuration("1h30m"), - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2w"}, + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99)) and apiserver_request_duration_seconds:burnrate30m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99))`), + For: monitoringDuration("1m"), + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99))`), + For: monitoringDuration("8m"), + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99))`), + For: monitoringDuration("30m"), + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99)) and apiserver_request_duration_seconds:burnrate2d{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99))`), + For: monitoringDuration("1h30m"), + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, }}, }, }, { @@ -1018,25 +1082,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d])) - sum by (resource, verb) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",le="0.1",resource=~"resource|",verb=~"LIST|GET"}[2d]))) / sum by (resource, verb) (rate(apiserver_request_duration_seconds_count{job="apiserver",resource=~"resource|",verb=~"LIST|GET"}[2d]))`), Labels: map[string]string{"job": "apiserver", "slo": "apiserver-read-resource-latency"}, }, { - Alert: "APIServerLatencyErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99)) and apiserver_request_duration_seconds:burnrate30m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99))`), - For: monitoringDuration("1m"), - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1d"}, - }, { - Alert: "APIServerLatencyErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99))`), - For: monitoringDuration("8m"), - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2d"}, - }, { - Alert: "APIServerLatencyErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99))`), - For: monitoringDuration("30m"), - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1w"}, - }, { - Alert: "APIServerLatencyErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99)) and apiserver_request_duration_seconds:burnrate2d{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99))`), - For: monitoringDuration("1h30m"), - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2w"}, + Alert: "APIServerLatencyErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99)) and apiserver_request_duration_seconds:burnrate30m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99))`), + For: monitoringDuration("1m"), + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "APIServerLatencyErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99))`), + For: monitoringDuration("8m"), + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "APIServerLatencyErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99))`), + For: monitoringDuration("30m"), + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, + }, { + Alert: "APIServerLatencyErrorBudgetBurn", + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99)) and apiserver_request_duration_seconds:burnrate2d{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99))`), + For: monitoringDuration("1h30m"), + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22apiserver-read-resource-latency%22%7D&from=now-1h&to=now&grouping=%7Bresource=%22{{$labels.resource}}%22,verb=%22{{$labels.verb}}%22%7D"}, }}, }, }, { @@ -1074,25 +1142,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum(count_over_time(up[4d])) - sum(sum_over_time(up[4d]))) / sum(count_over_time(up[4d]))`), Labels: map[string]string{"slo": "up-targets"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m"), - Expr: intstr.FromString(`up:burnrate5m{slo="up-targets"} > (14 * (1-0.99)) and up:burnrate1h{slo="up-targets"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m"), - Expr: intstr.FromString(`up:burnrate30m{slo="up-targets"} > (7 * (1-0.99)) and up:burnrate6h{slo="up-targets"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h"), - Expr: intstr.FromString(`up:burnrate2h{slo="up-targets"} > (2 * (1-0.99)) and up:burnrate1d{slo="up-targets"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h"), - Expr: intstr.FromString(`up:burnrate6h{slo="up-targets"} > (1 * (1-0.99)) and up:burnrate4d{slo="up-targets"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m"), + Expr: intstr.FromString(`up:burnrate5m{slo="up-targets"} > (14 * (1-0.99)) and up:burnrate1h{slo="up-targets"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m"), + Expr: intstr.FromString(`up:burnrate30m{slo="up-targets"} > (7 * (1-0.99)) and up:burnrate6h{slo="up-targets"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h"), + Expr: intstr.FromString(`up:burnrate2h{slo="up-targets"} > (2 * (1-0.99)) and up:burnrate1d{slo="up-targets"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h"), + Expr: intstr.FromString(`up:burnrate6h{slo="up-targets"} > (1 * (1-0.99)) and up:burnrate4d{slo="up-targets"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now"}, }}, }, }, { @@ -1130,25 +1202,29 @@ func TestObjective_Burnrates(t *testing.T) { Expr: intstr.FromString(`(sum by (instance, job) (count_over_time(up{instance!~"(127.0.0.1|localhost).*"}[4d])) - sum by (instance, job) (sum_over_time(up{instance!~"(127.0.0.1|localhost).*"}[4d]))) / sum by (instance, job) (count_over_time(up{instance!~"(127.0.0.1|localhost).*"}[4d]))`), Labels: map[string]string{"slo": "up-targets"}, }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("2m"), - Expr: intstr.FromString(`up:burnrate5m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (14 * (1-0.99)) and up:burnrate1h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets", "exhaustion": "2d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("15m"), - Expr: intstr.FromString(`up:burnrate30m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (7 * (1-0.99)) and up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m", "exhaustion": "4d"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("1h"), - Expr: intstr.FromString(`up:burnrate2h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2 * (1-0.99)) and up:burnrate1d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h", "exhaustion": "2w"}, - }, { - Alert: "ErrorBudgetBurn", - For: monitoringDuration("3h"), - Expr: intstr.FromString(`up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (1 * (1-0.99)) and up:burnrate4d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h", "exhaustion": "4w"}, + Alert: "ErrorBudgetBurn", + For: monitoringDuration("2m"), + Expr: intstr.FromString(`up:burnrate5m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (14 * (1-0.99)) and up:burnrate1h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (14 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets", "exhaustion": "2d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,instance=%22{{$labels.instance}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("15m"), + Expr: intstr.FromString(`up:burnrate30m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (7 * (1-0.99)) and up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (7 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m", "exhaustion": "4d"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,instance=%22{{$labels.instance}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("1h"), + Expr: intstr.FromString(`up:burnrate2h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2 * (1-0.99)) and up:burnrate1d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h", "exhaustion": "2w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,instance=%22{{$labels.instance}}%22%7D"}, + }, { + Alert: "ErrorBudgetBurn", + For: monitoringDuration("3h"), + Expr: intstr.FromString(`up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (1 * (1-0.99)) and up:burnrate4d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (1 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h", "exhaustion": "4w"}, + Annotations: map[string]string{"pyrra_url": "http://localhost:9090/objectives?expr=%7B__name__%3D%22up-targets%22%7D&from=now-1h&to=now&grouping=%7Bjob=%22{{$labels.job}}%22,instance=%22{{$labels.instance}}%22%7D"}, }}, }, }} @@ -1157,7 +1233,7 @@ func TestObjective_Burnrates(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - group, err := tc.slo.Burnrates() + group, err := tc.slo.Burnrates("http://localhost:9090") require.NoError(t, err) require.Equal(t, tc.rules, group) }) @@ -1377,9 +1453,9 @@ func TestObjective_IncreaseRules(t *testing.T) { Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "le": "1"}, // }, { // Alert: "SLOMetricAbsent", - // Expr: intstr.FromString(`absent(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}) == 1`), - // For: monitoringDuration("2m"), - // Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "severity": "critical"}, + // Expr: intstr.FromString(`absent(http_request_duration_seconds{code=~"2..",job="metrics-service-thanos-receive-default"}) == 1`), + // For: monitoringDuration("2m"), + // Labels: map[string]string{"job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "severity": "critical"}, }}, }, }, { @@ -2014,3 +2090,53 @@ func TestObjective_AlertNameMetricAbsent(t *testing.T) { }) } } + +func TestObjective_PyrraURLAnnotation(t *testing.T) { + tests := []struct { + name string + objective Objective + pyrraURL string + wantURL string + wantPresent bool + }{ + { + name: "with pyrra URL", + objective: objectiveHTTPRatio(), + pyrraURL: "https://pyrra.example.com", + wantURL: "https://pyrra.example.com/objectives/monitoring-http-errors", + wantPresent: true, + }, + { + name: "with pyrra URL and trailing slash", + objective: objectiveHTTPRatio(), + pyrraURL: "https://pyrra.example.com/", + wantURL: "https://pyrra.example.com/objectives/monitoring-http-errors", + wantPresent: true, + }, + { + name: "without pyrra URL", + objective: objectiveHTTPRatio(), + pyrraURL: "", + wantPresent: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + burnrates, err := tt.objective.Burnrates(tt.pyrraURL) + require.NoError(t, err) + + // Check that alerts have the pyrra_url annotation + for _, rule := range burnrates.Rules { + if rule.Alert != "" { + if tt.wantPresent { + require.Contains(t, rule.Annotations, "pyrra_url") + require.Equal(t, tt.wantURL, rule.Annotations["pyrra_url"]) + } else { + require.NotContains(t, rule.Annotations, "pyrra_url") + } + } + } + }) + } +}