added org id for multi tenant support

examples/grafana/detail.json

@@ -305,7 +305,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "((pyrra_availability{slo=\"$slo\"} - pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})",
+          "expr": "((pyrra_availability{slo=\"$slo\"} - on(slo) pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})",
           "instant": true,
           "legendFormat": "__auto",
           "range": false,
@@ -405,7 +405,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "((pyrra_availability{slo=\"$slo\"} - pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})",
+          "expr": "((pyrra_availability{slo=\"$slo\"} - on(slo) pyrra_objective{slo=\"$slo\"})) / (1 - pyrra_objective{slo=\"$slo\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"

examples/kubernetes/manifests/setup/pyrra-slo-CustomResourceDefinition.yaml

@@ -185,6 +185,9 @@ spec:
                   It represents the desired availability of the service in the given window.
                   float64 are not supported: https://github.com/kubernetes-sigs/controller-tools/issues/245
                 type: string
+              rulenamespace:
+                description: Recording rule namespace override, by default recording namespace and group are kept same.
+                type: string
               window:
                 description: Window within which the Target is supposed to be kept. Usually something like 1d, 7d or 28d.
                 type: string

go.mod

@@ -1,7 +1,9 @@
 module github.com/pyrra-dev/pyrra
 
+
 go 1.23
 
+
 require (
 	github.com/alecthomas/kong v0.9.0
 	github.com/bufbuild/connect-go v1.10.0

jsonnet/controller-gen/pyrra.dev_servicelevelobjectives.json

@@ -232,6 +232,10 @@
                     },
                     "type": "object"
                   },
+                  "rulenamespace": {
+                    "description": "Recording rule namespace override, by default recording namespace and group are kept same.",
+                    "type": "string"
+                  },
                   "target": {
                     "description": "Target is a string that's casted to a float64 between 0 - 100.\nIt represents the desired availability of the service in the given window.\nfloat64 are not supported: https://github.com/kubernetes-sigs/controller-tools/issues/245",
                     "type": "string"

kubernetes/api/v1alpha1/servicelevelobjective_types.go

@@ -90,6 +90,10 @@ type ServiceLevelObjectiveSpec struct {
 	// +optional
 	// Alerting customizes the alerting rules generated by Pyrra.
 	Alerting Alerting `json:"alerting"`
+
+	// +optional
+	// Recording rule namespace override, by default recording namespace and group are kept same.
+	RecordingRuleNamespace string `json:"rulenamespace"`
 }
 
 // ServiceLevelIndicator defines the underlying indicator that is a Prometheus metric.
@@ -558,13 +562,14 @@ func (in *ServiceLevelObjective) Internal() (slo.Objective, error) {
 	}
 
 	return slo.Objective{
-		Labels:      ls,
-		Annotations: in.Annotations,
-		Description: in.Spec.Description,
-		Target:      target / 100,
-		Window:      window,
-		Config:      string(config),
-		Alerting:    alerting,
+		Labels:                 ls,
+		Annotations:            in.Annotations,
+		Description:            in.Spec.Description,
+		Target:                 target / 100,
+		Window:                 window,
+		Config:                 string(config),
+		Alerting:               alerting,
+		RecordingRuleNamespace: in.Spec.RecordingRuleNamespace,
 		Indicator: slo.Indicator{
 			Ratio:         ratio,
 			Latency:       latency,

kubernetes/api/v1alpha1/servicelevelobjective_types_test.go

@@ -56,6 +56,7 @@ spec:
 				Burnrates: true,
 				Absent:    true,
 			},
+			RecordingRuleNamespace: "",
 			Indicator: slo.Indicator{
 				Ratio: &slo.RatioIndicator{
 					Errors: slo.Metric{
@@ -90,6 +91,7 @@ metadata:
 spec:
  target: 99.9
  window: 1w
+ rulenamespace: "test-namespace"
  indicator:
    ratio:
      errors:
@@ -109,6 +111,7 @@ spec:
 				Burnrates: true,
 				Absent:    true,
 			},
+			RecordingRuleNamespace: "test-namespace",
 			Indicator: slo.Indicator{
 				Ratio: &slo.RatioIndicator{
 					Errors: slo.Metric{

kubernetes/controllers/servicelevelobjective.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"time"
 
 	kitlog "github.com/go-kit/log"
@@ -35,6 +36,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/yaml"
 
 	pyrrav1alpha1 "github.com/pyrra-dev/pyrra/kubernetes/api/v1alpha1"
@@ -125,7 +128,6 @@ func (r *ServiceLevelObjectiveReconciler) reconcilePrometheusRule(ctx context.Co
 	}
 
 	newRule.ResourceVersion = rule.ResourceVersion
-
 	level.Info(logger).Log("msg", "updating prometheus rule", "namespace", rule.GetNamespace(), "name", rule.GetName())
 	if err := r.Update(ctx, newRule); err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to update prometheus rule: %w", err)
@@ -147,7 +149,13 @@ func (r *ServiceLevelObjectiveReconciler) reconcileMimirRuleGroup(ctx context.Co
 
 	level.Info(logger).Log("msg", "updating mimir rule", "name", newRuleGroup.Name)
 
-	err = r.MimirClient.SetRuleGroup(ctx, kubeObjective.GetName(), *newRuleGroup)
+	var ruleName string
+	if kubeObjective.Spec.RecordingRuleNamespace == "" {
+		ruleName = kubeObjective.GetName()
+	} else {
+		ruleName = kubeObjective.Spec.RecordingRuleNamespace
+	}
+	err = r.MimirClient.SetRuleGroup(ctx, ruleName, *newRuleGroup)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -160,8 +168,15 @@ func (r *ServiceLevelObjectiveReconciler) reconcileMimirRuleGroup(ctx context.Co
 	return ctrl.Result{}, nil
 }
 
+
 func (r *ServiceLevelObjectiveReconciler) deleteMimirRuleGroup(ctx context.Context, kubeObjective pyrrav1alpha1.ServiceLevelObjective) error {
-	return r.MimirClient.DeleteNamespace(ctx, kubeObjective.GetName())
+	var ruleName string
+	if kubeObjective.Spec.RecordingRuleNamespace == "" {
+		ruleName = kubeObjective.GetName()
+	} else {
+		ruleName = kubeObjective.Spec.RecordingRuleNamespace
+	}
+	return r.MimirClient.DeleteNamespace(ctx, ruleName)
 }
 
 func (r *ServiceLevelObjectiveReconciler) reconcileConfigMap(
@@ -210,6 +225,29 @@ func (r *ServiceLevelObjectiveReconciler) reconcileConfigMap(
 func (r *ServiceLevelObjectiveReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&pyrrav1alpha1.ServiceLevelObjective{}).
+		WithEventFilter(predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				oldObject := e.ObjectOld.(*pyrrav1alpha1.ServiceLevelObjective)
+				newObject := e.ObjectNew.(*pyrrav1alpha1.ServiceLevelObjective)
+
+				// Check if there's a difference; if not, skip reconciliation
+				if !reflect.DeepEqual(oldObject.Spec.RecordingRuleNamespace, newObject.Spec.RecordingRuleNamespace) {
+
+					var ruleNamespace string
+					if oldObject.Spec.RecordingRuleNamespace == "" {
+						ruleNamespace = oldObject.GetName()
+					} else {
+						ruleNamespace = oldObject.Spec.RecordingRuleNamespace
+					}
+
+					if err := r.MimirClient.DeleteNamespace(context.Background(), ruleNamespace); err != nil {
+						return false
+					}
+					return true
+				}
+				return true
+			},
+		}).
 		Complete(r)
 }
 

main.go

@@ -65,6 +65,7 @@ var CLI struct {
 		TLSCertFile                 string            `default:"" help:"File containing the default x509 Certificate for HTTPS."`
 		TLSPrivateKeyFile           string            `default:"" help:"File containing the default x509 private key matching --tls-cert-file."`
 		TLSClientCAFile             string            `default:"" help:"File containing the CA certificate for the client"`
+		OrgID                       string            `default:"" help:"Multi tenant authorization"`
 	} `cmd:"" help:"Runs Pyrra's API and UI."`
 	Filesystem struct {
 		ConfigFiles      string   `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use. Any non yaml files will be ignored."`
@@ -84,6 +85,8 @@ var CLI struct {
 		MimirBasicAuthUsername  string   `default:"" help:"The HTTP basic authentication username"`
 		MimirBasicAuthPassword  string   `default:"" help:"The HTTP basic authentication password"`
 		MimirWriteAlertingRules bool     `default:"false" help:"If alerting rules should be provisioned to the Mimir Ruler."`
+		MimirOrgID              string   `default:"" help:"Multi tenant authorization"`
+		MimirDeploymentMode     string   `default:"monolithic" help:"Mimir deployment mode it can be monolithic or microservices mode"`
 	} `cmd:"" help:"Runs Pyrra's Kubernetes operator and backend for the API."`
 	Generate struct {
 		ConfigFiles      string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."`
@@ -128,7 +131,15 @@ func main() {
 	if CLI.API.TLSClientCAFile != "" {
 		clientConfig.TLSConfig = promconfig.TLSConfig{CAFile: CLI.API.TLSClientCAFile}
 	}
-
+	if CLI.API.OrgID != "" {
+		clientConfig.HTTPHeaders = &promconfig.Headers{
+			Headers: map[string]promconfig.Header{
+				"X-Scope-OrgID": {
+					Values: []string{CLI.API.OrgID},
+				},
+			},
+		}
+	}
 	roundTripper, err := promconfig.NewRoundTripperFromConfig(clientConfig, "prometheus")
 	if err != nil {
 		level.Error(logger).Log("msg", "failed to create API client round tripper", "err", err)
@@ -162,6 +173,8 @@ func main() {
 			PrometheusPrefix:  CLI.Kubernetes.MimirPrometheusPrefix,
 			BasicAuthUsername: CLI.Kubernetes.MimirBasicAuthUsername,
 			BasicAuthPassword: CLI.Kubernetes.MimirBasicAuthPassword,
+			OrgID:             CLI.Kubernetes.MimirOrgID,
+			DeploymentMode:    CLI.Kubernetes.MimirDeploymentMode,
 		}
 
 		mimirClient, err = mimir.NewClient(mimirConfig)

mimir/client.go

@@ -13,6 +13,8 @@ type Client struct {
 	client           http.Client
 	address          *url.URL
 	prometheusPrefix string
+	orgID            string
+	deploymentMode   string
 }
 
 // Config is used to configure the client.
@@ -21,6 +23,8 @@ type Config struct {
 	PrometheusPrefix  string
 	BasicAuthUsername string
 	BasicAuthPassword string
+	OrgID             string
+	DeploymentMode    string
 }
 
 // NewClient creates a new client with the given configuration.
@@ -48,6 +52,8 @@ func NewClient(config Config) (*Client, error) {
 		client:           httpClient,
 		address:          addr,
 		prometheusPrefix: config.PrometheusPrefix,
+		orgID:            config.OrgID,
+		deploymentMode:   config.DeploymentMode,
 	}, nil
 }
 
@@ -66,13 +72,22 @@ func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error
 
 // Ready checks if mimir is ready to serve traffic.
 func (c *Client) Ready(ctx context.Context) error {
-	path := c.address.JoinPath("/ready")
+	path := c.address.JoinPath("/")
+	if c.deploymentMode == "monolithic" {
+		path = c.address.JoinPath("/ready")
+	} else if c.deploymentMode == "microservices" {
+		path = c.address.JoinPath("/api/v1/status/buildinfo")
+	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, path.String(), nil)
 	if err != nil {
 		return err
 	}
 
+	if c.orgID != "" {
+		req.Header.Set("X-Scope-OrgID", c.orgID)
+	}
+
 	resp, err := c.client.Do(req)
 	if err != nil {
 		return err

mimir/rulegroup.go

@@ -27,6 +27,10 @@ func (c *Client) SetRuleGroup(ctx context.Context, namespace string, ruleGroup r
 
 	req.Header.Set("Content-Type", "application/yaml")
 
+	if c.orgID != "" {
+		req.Header.Set("X-Scope-OrgID", c.orgID)
+	}
+
 	resp, err := c.client.Do(req)
 	if err != nil {
 		return err
@@ -49,6 +53,10 @@ func (c *Client) DeleteNamespace(ctx context.Context, namespace string) error {
 		return err
 	}
 
+	if c.orgID != "" {
+		req.Header.Set("X-Scope-OrgID", c.orgID)
+	}
+
 	resp, err := c.client.Do(req)
 	if err != nil {
 		return err

slo/rules.go

@@ -1170,6 +1170,7 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) {
 			return monitoringv1.RuleGroup{}, ErrGroupingUnsupported
 		}
 
+		ruleLabels := o.commonRuleLabels(sloName)
 		availability, err := parser.ParseExpr(`1 - sum(errorMetric{matchers="errors"} or vector(0)) / sum(metric{matchers="total"})`)
 		if err != nil {
 			return monitoringv1.RuleGroup{}, err
@@ -1225,6 +1226,12 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) {
 			errorMatchers: errorMatchers,
 		}.replace(availability)
 
+		for _, m := range totalMatchers {
+			if m.Type == labels.MatchEqual && m.Name != labels.MetricName {
+				ruleLabels[m.Name] = m.Value
+			}
+		}
+
 		rules = append(rules, monitoringv1.Rule{
 			Record: "pyrra_availability",
 			Expr:   intstr.FromString(availability.String()),
@@ -1270,6 +1277,8 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) {
 			return monitoringv1.RuleGroup{}, ErrGroupingUnsupported
 		}
 
+		ruleLabels := o.commonRuleLabels(sloName)
+
 		// availability
 		{
 			expr, err := parser.ParseExpr(`sum(errorMetric{matchers="errors"} or vector(0)) / sum(metric{matchers="total"})`)
@@ -1306,6 +1315,12 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) {
 				Value: o.Name(),
 			})
 
+			for _, m := range errorMatchers {
+				if m.Type == labels.MatchEqual && m.Name != labels.MetricName {
+					ruleLabels[m.Name] = m.Value
+				}
+			}
+
 			objectiveReplacer{
 				metric:        metric,
 				matchers:      matchers,
@@ -1390,6 +1405,7 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) {
 			return monitoringv1.RuleGroup{}, ErrGroupingUnsupported
 		}
 
+		ruleLabels := o.commonRuleLabels(sloName)
 		totalMetric := countName(o.Indicator.BoolGauge.Metric.Name, o.Window)
 		totalMatchers := cloneMatchers(o.Indicator.BoolGauge.Metric.LabelMatchers)
 		for _, m := range totalMatchers {
@@ -1418,6 +1434,11 @@ func (o Objective) GenericRules() (monitoringv1.RuleGroup, error) {
 			Value: o.Name(),
 		})
 
+		for _, m := range successMatchers {
+			if m.Type == labels.MatchEqual && m.Name != labels.MetricName {
+				ruleLabels[m.Name] = m.Value
+			}
+		}
 		// availability
 		{
 			expr, err := parser.ParseExpr(`sum(errorMetric{matchers="errors"}) / sum(metric{matchers="total"})`)