Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions filesystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"sync"
Expand Down Expand Up @@ -89,7 +90,7 @@ Objectives:
return objectives
}

func cmdFilesystem(logger log.Logger, reg *prometheus.Registry, promClient api.Client, configFiles, prometheusFolder string, genericRules bool) int {
func cmdFilesystem(logger log.Logger, reg *prometheus.Registry, promClient api.Client, configFiles, prometheusFolder string, genericRules bool, pyrraExternalURL *url.URL) int {
reconcilesTotal := prometheus.NewCounter(prometheus.CounterOpts{
Name: "pyrra_filesystem_reconciles_total",
Help: "The total amount of reconciles.",
Expand Down Expand Up @@ -179,7 +180,12 @@ func cmdFilesystem(logger log.Logger, reg *prometheus.Registry, promClient api.C
level.Debug(logger).Log("msg", "processing", "file", f)
reconcilesTotal.Inc()

err := writeRuleFile(logger, f, prometheusFolder, genericRules, false)
pyrraURL := ""
if pyrraExternalURL != nil {
pyrraURL = pyrraExternalURL.String()
}

err := writeRuleFile(logger, f, prometheusFolder, genericRules, false, pyrraURL)
if err != nil {
reconcilesErrors.Inc()
level.Error(logger).Log("msg", "error creating rule file", "file", f, "err", err)
Expand Down Expand Up @@ -301,7 +307,7 @@ func (s *FilesystemObjectiveServer) List(_ context.Context, req *connect.Request
}), nil
}

func writeRuleFile(logger log.Logger, file, prometheusFolder string, genericRules, operatorRule bool) error {
func writeRuleFile(logger log.Logger, file, prometheusFolder string, genericRules, operatorRule bool, externalURL string) error {
kubeObjective, objective, err := objectiveFromFile(file)
if err != nil {
return fmt.Errorf("failed to get objective: %w", err)
Expand All @@ -326,7 +332,7 @@ func writeRuleFile(logger log.Logger, file, prometheusFolder string, genericRule
return fmt.Errorf("failed to get increase rules: %w", err)
}

burnrates, err := objective.Burnrates()
burnrates, err := objective.Burnrates(externalURL)
if err != nil {
return fmt.Errorf("failed to get burn rate rules: %w", err)
}
Expand Down
10 changes: 8 additions & 2 deletions generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,21 +14,27 @@ limitations under the License.
package main

import (
"net/url"
"path/filepath"

"github.com/go-kit/log"
"github.com/go-kit/log/level"
)

func cmdGenerate(logger log.Logger, configFiles, prometheusFolder string, genericRules, operatorRule bool) int {
func cmdGenerate(logger log.Logger, configFiles, prometheusFolder string, genericRules, operatorRule bool, externalURL *url.URL) int {
filenames, err := filepath.Glob(configFiles)
if err != nil {
level.Error(logger).Log("msg", "getting file names", "err", err)
return 1
}

externalURLStr := ""
if externalURL != nil {
externalURLStr = externalURL.String()
}

for _, file := range filenames {
err := writeRuleFile(logger, file, prometheusFolder, genericRules, operatorRule)
err := writeRuleFile(logger, file, prometheusFolder, genericRules, operatorRule, externalURLStr)
if err != nil {
level.Error(logger).Log("msg", "generating rule files", "err", err)
return 1
Expand Down
22 changes: 16 additions & 6 deletions kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"net/http"
"net/url"
"os"
"syscall"
"time"
Expand Down Expand Up @@ -66,6 +67,7 @@ func cmdKubernetes(
certFile, privateKeyFile string,
mimirClient *mimir.Client,
mimirWriteAlertingRules bool,
pyrraExternalURL *url.URL,
) int {
setupLog := ctrl.Log.WithName("setup")
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
Expand All @@ -86,13 +88,19 @@ func cmdKubernetes(
os.Exit(1)
}

pyrraURL := ""
if pyrraExternalURL != nil {
pyrraURL = pyrraExternalURL.String()
}

reconciler := &controllers.ServiceLevelObjectiveReconciler{
Client: mgr.GetClient(),
Logger: log.With(logger, "controllers", "ServiceLevelObjective"),
GenericRules: genericRules,
ConfigMapMode: configMapMode,
MimirClient: mimirClient,
MimirWriteAlertingRules: mimirWriteAlertingRules,
PyrraExternalURL: pyrraURL,
}
if err = reconciler.SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ServiceLevelObjective")
Expand Down Expand Up @@ -129,7 +137,8 @@ func cmdKubernetes(
{
router := http.NewServeMux()
router.Handle(objectivesv1alpha1connect.NewObjectiveBackendServiceHandler(&KubernetesObjectiveServer{
client: mgr.GetClient(),
client: mgr.GetClient(),
pyrraURL: pyrraURL,
}))

server := http.Server{
Expand Down Expand Up @@ -166,7 +175,8 @@ type KubernetesClient interface {
}

type KubernetesObjectiveServer struct {
client KubernetesClient
client KubernetesClient
pyrraURL string
}

func (s *KubernetesObjectiveServer) List(ctx context.Context, req *connect.Request[objectivesv1alpha1.ListRequest]) (*connect.Response[objectivesv1alpha1.ListResponse], error) {
Expand Down Expand Up @@ -206,19 +216,19 @@ func (s *KubernetesObjectiveServer) List(ctx context.Context, req *connect.Reque
}

objectives := make([]*objectivesv1alpha1.Objective, 0, len(list.Items))
for _, s := range list.Items {
for _, slo := range list.Items {
if nameMatcher != nil {
if !nameMatcher.Matches(s.GetName()) {
if !nameMatcher.Matches(slo.GetName()) {
continue
}
}
if namespaceMatcher != nil {
if !namespaceMatcher.Matches(s.GetNamespace()) {
if !namespaceMatcher.Matches(slo.GetNamespace()) {
continue
}
}

internal, err := s.Internal()
internal, err := slo.Internal()
if err != nil {
return nil, connect.NewError(connect.CodeInternal, err)
}
Expand Down
19 changes: 10 additions & 9 deletions kubernetes/controllers/servicelevelobjective.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ type ServiceLevelObjectiveReconciler struct {
MimirWriteAlertingRules bool
Logger kitlog.Logger
Scheme *runtime.Scheme
PyrraExternalURL string
ConfigMapMode bool
GenericRules bool
}
Expand Down Expand Up @@ -106,7 +107,7 @@ func (r *ServiceLevelObjectiveReconciler) Reconcile(ctx context.Context, req ctr
}

func (r *ServiceLevelObjectiveReconciler) reconcilePrometheusRule(ctx context.Context, logger kitlog.Logger, req ctrl.Request, kubeObjective pyrrav1alpha1.ServiceLevelObjective) (ctrl.Result, error) {
newRule, err := makePrometheusRule(kubeObjective, r.GenericRules)
newRule, err := makePrometheusRule(kubeObjective, r.GenericRules, r.PyrraExternalURL)
if err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -139,7 +140,7 @@ func (r *ServiceLevelObjectiveReconciler) reconcilePrometheusRule(ctx context.Co
}

func (r *ServiceLevelObjectiveReconciler) reconcileMimirRuleGroup(ctx context.Context, logger kitlog.Logger, kubeObjective pyrrav1alpha1.ServiceLevelObjective) (ctrl.Result, error) {
newRuleGroup, err := makeMimirRuleGroup(kubeObjective, r.GenericRules, r.MimirWriteAlertingRules)
newRuleGroup, err := makeMimirRuleGroup(kubeObjective, r.GenericRules, r.MimirWriteAlertingRules, r.PyrraExternalURL)
if err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -171,7 +172,7 @@ func (r *ServiceLevelObjectiveReconciler) reconcileConfigMap(
) (ctrl.Result, error) {
name := fmt.Sprintf("pyrra-recording-rule-%s", kubeObjective.GetName())

newConfigMap, err := makeConfigMap(name, kubeObjective, r.GenericRules)
newConfigMap, err := makeConfigMap(name, kubeObjective, r.GenericRules, r.PyrraExternalURL)
if err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -218,7 +219,7 @@ func (r *ServiceLevelObjectiveReconciler) SetupWebhookWithManager(mgr ctrl.Manag
Complete()
}

func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool) (*corev1.ConfigMap, error) {
func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool, externalURL string) (*corev1.ConfigMap, error) {
objective, err := kubeObjective.Internal()
if err != nil {
return nil, fmt.Errorf("failed to get objective: %w", err)
Expand All @@ -228,7 +229,7 @@ func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjectiv
if err != nil {
return nil, fmt.Errorf("failed to get increase rules: %w", err)
}
burnrates, err := objective.Burnrates()
burnrates, err := objective.Burnrates(externalURL)
if err != nil {
return nil, fmt.Errorf("failed to get burn rate rules: %w", err)
}
Expand Down Expand Up @@ -286,7 +287,7 @@ func makeConfigMap(name string, kubeObjective pyrrav1alpha1.ServiceLevelObjectiv
}, nil
}

func makeMimirRuleGroup(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules, writeAlertingRules bool) (*rulefmt.RuleGroup, error) {
func makeMimirRuleGroup(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules, writeAlertingRules bool, externalURL string) (*rulefmt.RuleGroup, error) {
objective, err := kubeObjective.Internal()
if err != nil {
return nil, fmt.Errorf("failed to get objective: %w", err)
Expand All @@ -298,7 +299,7 @@ func makeMimirRuleGroup(kubeObjective pyrrav1alpha1.ServiceLevelObjective, gener
}
increasesMimirRules := prometheusRulesToMimirRules(increases.Rules, writeAlertingRules)

burnrates, err := objective.Burnrates()
burnrates, err := objective.Burnrates(externalURL)
if err != nil {
return nil, fmt.Errorf("failed to get burn rate rules: %w", err)
}
Expand Down Expand Up @@ -372,7 +373,7 @@ func prometheusRulesToMimirRules(promRules []monitoringv1.Rule, writeAlertingRul
return rules
}

func makePrometheusRule(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool) (*monitoringv1.PrometheusRule, error) {
func makePrometheusRule(kubeObjective pyrrav1alpha1.ServiceLevelObjective, genericRules bool, externalURL string) (*monitoringv1.PrometheusRule, error) {
objective, err := kubeObjective.Internal()
if err != nil {
return nil, fmt.Errorf("failed to get objective: %w", err)
Expand All @@ -382,7 +383,7 @@ func makePrometheusRule(kubeObjective pyrrav1alpha1.ServiceLevelObjective, gener
if err != nil {
return nil, fmt.Errorf("failed to get increase rules: %w", err)
}
burnrates, err := objective.Burnrates()
burnrates, err := objective.Burnrates(externalURL)
if err != nil {
return nil, fmt.Errorf("failed to get burn rate rules: %w", err)
}
Expand Down
34 changes: 20 additions & 14 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ var CLI struct {
PrometheusURL *url.URL `default:"http://localhost:9090" help:"The URL to the Prometheus to query."`
PrometheusFolder string `default:"/etc/prometheus/pyrra/" help:"The folder where Pyrra writes the generates Prometheus rules and alerts."`
GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."`
ExternalURL *url.URL `default:"http://localhost:9099" help:"The URL for Pyrra to be included in alert annotations. This will be used to generate direct links to the Pyrra UI in alerts."`
} `cmd:"" help:"Runs Pyrra's filesystem operator and backend for the API."`
Kubernetes struct {
MetricsAddr string `default:":8080" help:"The address the metric endpoint binds to."`
Expand All @@ -88,12 +89,14 @@ var CLI struct {
MimirBasicAuthUsername string `default:"" help:"The HTTP basic authentication username"`
MimirBasicAuthPassword string `default:"" help:"The HTTP basic authentication password"`
MimirWriteAlertingRules bool `default:"false" help:"If alerting rules should be provisioned to the Mimir Ruler."`
ExternalURL *url.URL `default:"http://localhost:9099" help:"The URL for Pyrra to be included in alert annotations. This will be used to generate direct links to the Pyrra UI in alerts."`
} `cmd:"" help:"Runs Pyrra's Kubernetes operator and backend for the API."`
Generate struct {
ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."`
PrometheusFolder string `default:"/etc/prometheus/pyrra/" help:"The folder where Pyrra writes the generated Prometheus rules and alerts."`
GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."`
OperatorRule bool `default:"false" help:"Generate rule files as prometheus-operator PrometheusRule: https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PrometheusRule."`
ConfigFiles string `default:"/etc/pyrra/*.yaml" help:"The folder where Pyrra finds the config files to use."`
PrometheusFolder string `default:"/etc/prometheus/pyrra/" help:"The folder where Pyrra writes the generated Prometheus rules and alerts."`
GenericRules bool `default:"false" help:"Enabled generic recording rules generation to make it easier for tools like Grafana."`
OperatorRule bool `default:"false" help:"Generate rule files as prometheus-operator PrometheusRule: https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PrometheusRule."`
ExternalURL *url.URL `default:"http://localhost:9099" help:"The URL for Pyrra to be included in alert annotations. This will be used to generate direct links to the Pyrra UI in alerts."`
} `cmd:"" help:"Read SLO config files and rewrites them as Prometheus rules and alerts."`
}

Expand Down Expand Up @@ -162,21 +165,21 @@ func main() {
level.Info(logger).Log("msg", "using Prometheus", "url", prometheusURL.String())

// Default external url to prometheus if not defined
externalURL := prometheusURL
externalDatasourceURL := prometheusURL
if CLI.API.PrometheusExternalURL != nil && CLI.API.GrafanaExternalURL != nil {
level.Error(logger).Log("msg", "prometheus external URL set alongside grafana external url")
os.Exit(1)
} else if CLI.API.GrafanaExternalURL == nil && CLI.API.GrafanaExternalDatasourceID != "" {
level.Error(logger).Log("msg", "grafana external datasource id set without grafana external url")
os.Exit(1)
} else if CLI.API.PrometheusExternalURL != nil {
externalURL = CLI.API.PrometheusExternalURL
externalDatasourceURL = CLI.API.PrometheusExternalURL
} else if CLI.API.GrafanaExternalURL != nil {
if CLI.API.GrafanaExternalDatasourceID == "" {
level.Error(logger).Log("msg", "grafana external datasource id cannot be empty when using an external grafana url")
os.Exit(1)
}
externalURL = CLI.API.GrafanaExternalURL
externalDatasourceURL = CLI.API.GrafanaExternalURL
}

// Mimir Client
Expand Down Expand Up @@ -211,7 +214,7 @@ func main() {
logger,
reg,
client,
externalURL,
externalDatasourceURL,
CLI.API.APIURL,
CLI.API.GrafanaExternalOrgID,
CLI.API.GrafanaExternalDatasourceID,
Expand All @@ -228,6 +231,7 @@ func main() {
CLI.Filesystem.ConfigFiles,
CLI.Filesystem.PrometheusFolder,
CLI.Filesystem.GenericRules,
CLI.Filesystem.ExternalURL,
)
case "kubernetes":
code = cmdKubernetes(
Expand All @@ -240,6 +244,7 @@ func main() {
CLI.Kubernetes.TLSPrivateKeyFile,
mimirClient,
CLI.Kubernetes.MimirWriteAlertingRules,
CLI.Kubernetes.ExternalURL,
)
case "generate":
code = cmdGenerate(
Expand All @@ -248,6 +253,7 @@ func main() {
CLI.Generate.PrometheusFolder,
CLI.Generate.GenericRules,
CLI.Generate.OperatorRule,
CLI.Generate.ExternalURL,
)
}
os.Exit(code)
Expand All @@ -257,7 +263,7 @@ func cmdAPI(
logger log.Logger,
reg *prometheus.Registry,
promClient api.Client,
externalURL, apiURL *url.URL,
externalDatasourceURL, apiURL *url.URL,
externalGrafanaOrgID, externalGrafanaDatasourceID string,
routePrefix, uiRoutePrefix string,
tlsCertFile, tlsPrivateKeyFile string,
Expand All @@ -277,9 +283,9 @@ func cmdAPI(
}

if externalGrafanaDatasourceID == "" {
level.Info(logger).Log("msg", "UI redirect to Prometheus", "url", externalURL.String())
level.Info(logger).Log("msg", "UI redirect to Prometheus", "url", externalDatasourceURL.String())
} else {
level.Info(logger).Log("msg", "UI redirect to Grafana", "url", externalURL.String(),
level.Info(logger).Log("msg", "UI redirect to Grafana", "url", externalDatasourceURL.String(),
"datasourceId", externalGrafanaDatasourceID, "orgId", externalGrafanaOrgID)
}
level.Info(logger).Log("msg", "using API at", "url", apiURL.String())
Expand Down Expand Up @@ -370,13 +376,13 @@ func cmdAPI(
r.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
r.Get("/objectives", func(w http.ResponseWriter, _ *http.Request) {
err := tmpl.Execute(w, struct {
ExternalURL string
ExternalDatasourceURL string
ExternalGrafanaDatasourceID string
ExternalGrafanaOrgID string
PathPrefix string
APIBasepath string
}{
ExternalURL: externalURL.String(),
ExternalDatasourceURL: externalDatasourceURL.String(),
ExternalGrafanaDatasourceID: externalGrafanaDatasourceID,
ExternalGrafanaOrgID: externalGrafanaOrgID,
PathPrefix: uiRoutePrefix,
Expand All @@ -396,7 +402,7 @@ func cmdAPI(
PathPrefix string
APIBasepath string
}{
ExternalURL: externalURL.String(),
ExternalURL: externalDatasourceURL.String(),
ExternalGrafanaDatasourceID: externalGrafanaDatasourceID,
ExternalGrafanaOrgID: externalGrafanaOrgID,
PathPrefix: uiRoutePrefix,
Expand Down
Loading
Loading