From 78c1d61de3ae29e90b0d1784505c8973cc39ce0d Mon Sep 17 00:00:00 2001 From: Vaishnavi Hire Date: Wed, 21 Jun 2023 09:36:03 -0400 Subject: [PATCH] Deploy Monitoring resources --- .../dscinitialization_controller.go | 18 ++ controllers/dscinitialization/monitoring.go | 269 ++++++++++++++++++ controllers/dscinitialization/utils.go | 35 +++ go.mod | 3 +- go.sum | 2 + main.go | 1 + 6 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 controllers/dscinitialization/monitoring.go diff --git a/controllers/dscinitialization/dscinitialization_controller.go b/controllers/dscinitialization/dscinitialization_controller.go index ae3e6955c27..f7087fd3868 100644 --- a/controllers/dscinitialization/dscinitialization_controller.go +++ b/controllers/dscinitialization/dscinitialization_controller.go @@ -38,6 +38,11 @@ import ( "github.com/opendatahub-io/opendatahub-operator/pkg/deploy" ) +const ( + managedServiceApplicationNamespace = "redhat-ods-applications" + defaultManifestPath = "/opt/odh-manifests" +) + // DSCInitializationReconciler reconciles a DSCInitialization object type DSCInitializationReconciler struct { client.Client @@ -112,6 +117,19 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re } } + // If monitoring enabled + if instance.Spec.Monitoring.Enabled { + if r.isManagedService() { + err := r.configureManagedMonitoring(instance) + if err != nil { + return reconcile.Result{}, err + } + + } else { + // TODO: ODH specific monitoring logic + } + } + // Finish reconciling reason := status.ReconcileCompleted message := status.ReconcileCompletedMessage diff --git a/controllers/dscinitialization/monitoring.go b/controllers/dscinitialization/monitoring.go new file mode 100644 index 00000000000..dbcce565177 --- /dev/null +++ b/controllers/dscinitialization/monitoring.go @@ -0,0 +1,269 @@ +package dscinitialization + +import ( + "context" + "crypto/sha256" + b64 "encoding/base64" + "fmt" + "strings" + + routev1 "github.com/openshift/api/route/v1" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + dsci "github.com/opendatahub-io/opendatahub-operator/apis/dscinitialization/v1alpha1" + "github.com/opendatahub-io/opendatahub-operator/pkg/deploy" +) + +func configurePrometheus(dsciInit *dsci.DSCInitialization, r *DSCInitializationReconciler) error { + // Get alertmanager host + alertmanagerRoute := &routev1.Route{} + err := r.Client.Get(context.TODO(), client.ObjectKey{ + Namespace: dsciInit.Spec.Monitoring.Namespace, + Name: "alertmanager", + }, alertmanagerRoute) + + if err != nil { + return fmt.Errorf("error getting alertmanager host : %v", err) + } + + alertManagerConfigMap := &corev1.ConfigMap{} + err = r.Client.Get(context.TODO(), client.ObjectKey{ + Namespace: dsciInit.Spec.Monitoring.Namespace, + Name: "alertmanager", + }, alertManagerConfigMap) + + if err != nil { + return fmt.Errorf("error getting alertmanager configmap : %v", err) + } + + prometheusConfigMap := &corev1.ConfigMap{} + err = r.Client.Get(context.TODO(), client.ObjectKey{ + Namespace: dsciInit.Spec.Monitoring.Namespace, + Name: "prometheus", + }, prometheusConfigMap) + + if err != nil { + return fmt.Errorf("error getting prometheus configmap : %v", err) + } + + alertmanagerData, err := getMonitoringData(alertManagerConfigMap.Data["alertmanager.yml"]) + if err != nil { + return err + } + + prometheusData, err := getMonitoringData(fmt.Sprint(prometheusConfigMap.Data)) + if err != nil { + return err + } + + // Update prometheus manifests + err = ReplaceStringsInFile(defaultManifestPath+"/monitoring/prometheus/prometheus.yaml", map[string]string{ + "": alertmanagerRoute.Spec.Host, + "": alertmanagerData, + "": prometheusData, + }) + if err != nil { + return err + } + + err = ReplaceStringsInFile(defaultManifestPath+"/monitoring/prometheus/prometheus-viewer-rolebinding.yaml", map[string]string{ + "": dsciInit.Spec.Monitoring.Namespace, + }) + + if err != nil { + return err + } + + // Deploy manifests + err = deploy.DeployManifestsFromPath(dsciInit, r.Client, + defaultManifestPath+"/monitoring/prometheus", + dsciInit.Spec.Monitoring.Namespace, r.Scheme) + if err != nil { + return err + } + + // Create proxy secret + if err := createMonitoringProxySecret("prometheus-proxy", dsciInit, r.Client, r.Scheme); err != nil { + return err + } + return nil +} + +func configureAlertManager(dsciInit *dsci.DSCInitialization, r *DSCInitializationReconciler) error { + // Get Deadmansnitch secret + deadmansnitchSecret, err := r.waitForManagedSecret("redhat-rhods-deadmanssnitch", dsciInit.Spec.Monitoring.Namespace) + if err != nil { + return fmt.Errorf("error getting deadmansnitch secret: %v", err) + } + + // Get PagerDuty Secret + pagerDutySecret, err := r.waitForManagedSecret("redhat-rhods-pagerduty", dsciInit.Spec.Monitoring.Namespace) + if err != nil { + return fmt.Errorf("error getting pagerduty secret: %v", err) + } + + // Get Smtp Secret + smtpSecret, err := r.waitForManagedSecret("redhat-rhods-smtp", dsciInit.Spec.Monitoring.Namespace) + if err != nil { + return fmt.Errorf("error getting smtp secret: %v", err) + } + + // Replace variables in alertmanager configmap + // TODO: Following variables can later be exposed by the API + err = ReplaceStringsInFile(defaultManifestPath+"/monitoring/alertmanager/monitoring-configs.yaml", + map[string]string{ + "": b64.StdEncoding.EncodeToString(deadmansnitchSecret.Data["SNITCH_URL"]), + "": b64.StdEncoding.EncodeToString(pagerDutySecret.Data["PAGERDUTY_KEY"]), + "": b64.StdEncoding.EncodeToString(smtpSecret.Data["host"]), + "": b64.StdEncoding.EncodeToString(smtpSecret.Data["port"]), + "": b64.StdEncoding.EncodeToString(smtpSecret.Data["username"]), + "": b64.StdEncoding.EncodeToString(smtpSecret.Data["password"]), + }) + + if err != nil { + return err + } + + err = deploy.DeployManifestsFromPath(dsciInit, r.Client, + defaultManifestPath+"/monitoring/alertmanager", + dsciInit.Spec.Monitoring.Namespace, r.Scheme) + if err != nil { + return err + } + + // TODO: Add watch for SMTP secret and configure emails + + // Create proxy secret + if err := createMonitoringProxySecret("alertmanager-proxy", dsciInit, r.Client, r.Scheme); err != nil { + return err + } + return nil +} + +func configureBlackboxExporter(dsciInit *dsci.DSCInitialization, cli client.Client, s *runtime.Scheme) error { + + consoleRoute := &routev1.Route{} + err := cli.Get(context.TODO(), client.ObjectKey{Name: "console", Namespace: "openshift-console"}, consoleRoute) + if err != nil { + if !apierrs.IsNotFound(err) { + return err + } + } + + if apierrs.IsNotFound(err) || strings.Contains(consoleRoute.Spec.Host, "redhat.com") { + err := deploy.DeployManifestsFromPath(dsciInit, cli, + defaultManifestPath+"/monitoring/blackbox-exporter/internal", + dsciInit.Spec.Monitoring.Namespace, s) + if err != nil { + return err + } + + } else { + err := deploy.DeployManifestsFromPath(dsciInit, cli, + defaultManifestPath+"/monitoring/blackbox-exporter/external", + dsciInit.Spec.Monitoring.Namespace, s) + if err != nil { + return err + } + } + return nil +} + +func (r *DSCInitializationReconciler) configureManagedMonitoring(dscInit *dsci.DSCInitialization) error { + // configure Alertmanager + if err := configureAlertManager(dscInit, r); err != nil { + fmt.Printf("Error in alertmanager") + return err + } + + // configure Prometheus + if err := configurePrometheus(dscInit, r); err != nil { + fmt.Printf("Error in prometheus") + return err + } + + // configure Blackbox exporter + if err := configureBlackboxExporter(dscInit, r.Client, r.Scheme); err != nil { + fmt.Printf("Error in blackbox exporter") + return err + } + return nil +} + +func createMonitoringProxySecret(name string, dsciInit *dsci.DSCInitialization, cli client.Client, s *runtime.Scheme) error { + + sessionSecret, err := GenerateRandomHex(32) + if err != nil { + return err + } + + desiredProxySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: dsciInit.Spec.Monitoring.Namespace, + }, + Data: map[string][]byte{ + "session_secret": []byte(b64.StdEncoding.EncodeToString(sessionSecret)), + }, + } + + foundProxySecret := &corev1.Secret{} + err = cli.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: dsciInit.Spec.Monitoring.Namespace}, foundProxySecret) + if err != nil { + if apierrs.IsNotFound(err) { + // Set Controller reference + err = ctrl.SetControllerReference(dsciInit, desiredProxySecret, s) + if err != nil { + return err + } + err = cli.Create(context.TODO(), desiredProxySecret) + if err != nil && !apierrs.IsAlreadyExists(err) { + return err + } + } else { + return err + } + } + return nil + +} + +func replaceInAlertManagerConfigmap(cli client.Client, dsciInit *dsci.DSCInitialization, cmName, replaceVariable, replaceValue string) error { + prometheusConfig := &corev1.ConfigMap{} + err := cli.Get(context.TODO(), client.ObjectKey{Name: cmName, Namespace: dsciInit.Spec.Monitoring.Namespace}, prometheusConfig) + if err != nil { + if apierrs.IsNotFound(err) { + return nil + } + return err + } + prometheusAlertmanagerContent := prometheusConfig.Data["alertmanager.yml"] + prometheusAlertmanagerContent = strings.ReplaceAll(prometheusAlertmanagerContent, replaceVariable, replaceValue) + + prometheusConfig.Data["alertmanager.yml"] = prometheusAlertmanagerContent + return cli.Update(context.TODO(), prometheusConfig) +} + +func getMonitoringData(data string) (string, error) { + // Create a new SHA-256 hash object + hash := sha256.New() + + // Write the input data to the hash object + _, err := hash.Write([]byte(data)) + if err != nil { + return "", err + } + + // Get the computed hash sum + hashSum := hash.Sum(nil) + + // Encode the hash sum to Base64 + encodedData := b64.StdEncoding.EncodeToString(hashSum) + + return encodedData, nil +} diff --git a/controllers/dscinitialization/utils.go b/controllers/dscinitialization/utils.go index 591983a2a55..3b5e4164fcf 100644 --- a/controllers/dscinitialization/utils.go +++ b/controllers/dscinitialization/utils.go @@ -9,18 +9,25 @@ import ( "k8s.io/client-go/util/retry" "reflect" "strings" + "time" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" authv1 "k8s.io/api/rbac/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" dsci "github.com/opendatahub-io/opendatahub-operator/apis/dscinitialization/v1alpha1" ) +var ( + resourceInterval = 10 * time.Second + resourceTimeout = 1 * time.Minute +) + // createOdhNamespace creates a Namespace with given name and with ODH defaults. The defaults include: // - Odh specific labels // - Pod security labels for baseline permissions @@ -58,6 +65,12 @@ func (r *DSCInitializationReconciler) createOdhNamespace(dscInit *dsci.DSCInitia r.Log.Error(err, "Unable to fetch namespace", "name", name) return err } + } else if dscInit.Spec.Monitoring.Enabled { + err = r.Patch(ctx, foundNamespace, client.RawPatch(types.MergePatchType, + []byte(`{"metadata": {"labels": {"openshift.io/cluster-monitoring": "true"}}}`))) + if err != nil { + return err + } } // Create default NetworkPolicy for the namespace @@ -202,6 +215,28 @@ func CompareNotebookNetworkPolicies(np1 netv1.NetworkPolicy, np2 netv1.NetworkPo reflect.DeepEqual(np1.Spec, np2.Spec) } +func (r *DSCInitializationReconciler) waitForManagedSecret(name, namespace string) (*corev1.Secret, error) { + managedSecret := &corev1.Secret{} + err := wait.Poll(resourceInterval, resourceTimeout, func() (done bool, err error) { + + err = r.Client.Get(context.TODO(), client.ObjectKey{ + Namespace: namespace, + Name: name, + }, managedSecret) + + if err != nil { + if apierrs.IsNotFound(err) { + return false, nil + } + return false, err + } else { + return true, nil + } + }) + + return managedSecret, err +} + func GenerateRandomHex(length int) ([]byte, error) { // Calculate the required number of bytes numBytes := length / 2 diff --git a/go.mod b/go.mod index 2384b851cbe..7ecff5e0206 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,10 @@ require ( github.com/onsi/ginkgo/v2 v2.6.0 github.com/onsi/gomega v1.24.1 github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd + github.com/openshift/api v3.9.0+incompatible github.com/openshift/custom-resource-status v1.1.2 k8s.io/api v0.26.2 + k8s.io/apiextensions-apiserver v0.26.1 k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.1 sigs.k8s.io/controller-runtime v0.14.4 @@ -70,7 +72,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.1 // indirect k8s.io/component-base v0.26.1 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20230109183929-3758b55a6596 // indirect diff --git a/go.sum b/go.sum index 1dde00ce6a6..3f5a44dfd88 100644 --- a/go.sum +++ b/go.sum @@ -271,6 +271,8 @@ github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd h1:6elrLdOa+BRHJVaHnZAHltufWk0pzPZYF67fX9aFCjU= github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd/go.mod h1:cDMtOZx741HfmmUMmT09PWM8cOBxEJp3ipUHeHPr8F4= +github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= +github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/main.go b/main.go index 8a3213f6441..ed3f64bd680 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ func init() { utilruntime.Must(apiextv1.AddToScheme(scheme)) utilruntime.Must(routev1.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme }