Skip to content

Commit

Permalink
Deploy Monitoring resources
Browse files Browse the repository at this point in the history
  • Loading branch information
VaishnaviHire committed Jun 23, 2023
1 parent d789a38 commit 78c1d61
Show file tree
Hide file tree
Showing 6 changed files with 327 additions and 1 deletion.
18 changes: 18 additions & 0 deletions controllers/dscinitialization/dscinitialization_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ import (
"github.com/opendatahub-io/opendatahub-operator/pkg/deploy"
)

const (
managedServiceApplicationNamespace = "redhat-ods-applications"
defaultManifestPath = "/opt/odh-manifests"
)

// DSCInitializationReconciler reconciles a DSCInitialization object
type DSCInitializationReconciler struct {
client.Client
Expand Down Expand Up @@ -112,6 +117,19 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re
}
}

// If monitoring enabled
if instance.Spec.Monitoring.Enabled {
if r.isManagedService() {
err := r.configureManagedMonitoring(instance)
if err != nil {
return reconcile.Result{}, err
}

} else {
// TODO: ODH specific monitoring logic
}
}

// Finish reconciling
reason := status.ReconcileCompleted
message := status.ReconcileCompletedMessage
Expand Down
269 changes: 269 additions & 0 deletions controllers/dscinitialization/monitoring.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,269 @@
package dscinitialization

import (
"context"
"crypto/sha256"
b64 "encoding/base64"
"fmt"
"strings"

routev1 "github.com/openshift/api/route/v1"
corev1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

dsci "github.com/opendatahub-io/opendatahub-operator/apis/dscinitialization/v1alpha1"
"github.com/opendatahub-io/opendatahub-operator/pkg/deploy"
)

func configurePrometheus(dsciInit *dsci.DSCInitialization, r *DSCInitializationReconciler) error {
// Get alertmanager host
alertmanagerRoute := &routev1.Route{}
err := r.Client.Get(context.TODO(), client.ObjectKey{
Namespace: dsciInit.Spec.Monitoring.Namespace,
Name: "alertmanager",
}, alertmanagerRoute)

if err != nil {
return fmt.Errorf("error getting alertmanager host : %v", err)
}

alertManagerConfigMap := &corev1.ConfigMap{}
err = r.Client.Get(context.TODO(), client.ObjectKey{
Namespace: dsciInit.Spec.Monitoring.Namespace,
Name: "alertmanager",
}, alertManagerConfigMap)

if err != nil {
return fmt.Errorf("error getting alertmanager configmap : %v", err)
}

prometheusConfigMap := &corev1.ConfigMap{}
err = r.Client.Get(context.TODO(), client.ObjectKey{
Namespace: dsciInit.Spec.Monitoring.Namespace,
Name: "prometheus",
}, prometheusConfigMap)

if err != nil {
return fmt.Errorf("error getting prometheus configmap : %v", err)
}

alertmanagerData, err := getMonitoringData(alertManagerConfigMap.Data["alertmanager.yml"])
if err != nil {
return err
}

prometheusData, err := getMonitoringData(fmt.Sprint(prometheusConfigMap.Data))
if err != nil {
return err
}

// Update prometheus manifests
err = ReplaceStringsInFile(defaultManifestPath+"/monitoring/prometheus/prometheus.yaml", map[string]string{
"<set_alertmanager_host>": alertmanagerRoute.Spec.Host,
"<alertmanager_config_hash>": alertmanagerData,
"<prometheus_config_hash>": prometheusData,
})
if err != nil {
return err
}

err = ReplaceStringsInFile(defaultManifestPath+"/monitoring/prometheus/prometheus-viewer-rolebinding.yaml", map[string]string{
"<odh_monitoring_project>": dsciInit.Spec.Monitoring.Namespace,
})

if err != nil {
return err
}

// Deploy manifests
err = deploy.DeployManifestsFromPath(dsciInit, r.Client,
defaultManifestPath+"/monitoring/prometheus",
dsciInit.Spec.Monitoring.Namespace, r.Scheme)
if err != nil {
return err
}

// Create proxy secret
if err := createMonitoringProxySecret("prometheus-proxy", dsciInit, r.Client, r.Scheme); err != nil {
return err
}
return nil
}

func configureAlertManager(dsciInit *dsci.DSCInitialization, r *DSCInitializationReconciler) error {
// Get Deadmansnitch secret
deadmansnitchSecret, err := r.waitForManagedSecret("redhat-rhods-deadmanssnitch", dsciInit.Spec.Monitoring.Namespace)
if err != nil {
return fmt.Errorf("error getting deadmansnitch secret: %v", err)
}

// Get PagerDuty Secret
pagerDutySecret, err := r.waitForManagedSecret("redhat-rhods-pagerduty", dsciInit.Spec.Monitoring.Namespace)
if err != nil {
return fmt.Errorf("error getting pagerduty secret: %v", err)
}

// Get Smtp Secret
smtpSecret, err := r.waitForManagedSecret("redhat-rhods-smtp", dsciInit.Spec.Monitoring.Namespace)
if err != nil {
return fmt.Errorf("error getting smtp secret: %v", err)
}

// Replace variables in alertmanager configmap
// TODO: Following variables can later be exposed by the API
err = ReplaceStringsInFile(defaultManifestPath+"/monitoring/alertmanager/monitoring-configs.yaml",
map[string]string{
"<snitch_url>": b64.StdEncoding.EncodeToString(deadmansnitchSecret.Data["SNITCH_URL"]),
"<pagerduty_token>": b64.StdEncoding.EncodeToString(pagerDutySecret.Data["PAGERDUTY_KEY"]),
"<smtp_host>": b64.StdEncoding.EncodeToString(smtpSecret.Data["host"]),
"<smtp_port>": b64.StdEncoding.EncodeToString(smtpSecret.Data["port"]),
"<smtp_username>": b64.StdEncoding.EncodeToString(smtpSecret.Data["username"]),
"<smtp_password>": b64.StdEncoding.EncodeToString(smtpSecret.Data["password"]),
})

if err != nil {
return err
}

err = deploy.DeployManifestsFromPath(dsciInit, r.Client,
defaultManifestPath+"/monitoring/alertmanager",
dsciInit.Spec.Monitoring.Namespace, r.Scheme)
if err != nil {
return err
}

// TODO: Add watch for SMTP secret and configure emails

// Create proxy secret
if err := createMonitoringProxySecret("alertmanager-proxy", dsciInit, r.Client, r.Scheme); err != nil {
return err
}
return nil
}

func configureBlackboxExporter(dsciInit *dsci.DSCInitialization, cli client.Client, s *runtime.Scheme) error {

consoleRoute := &routev1.Route{}
err := cli.Get(context.TODO(), client.ObjectKey{Name: "console", Namespace: "openshift-console"}, consoleRoute)
if err != nil {
if !apierrs.IsNotFound(err) {
return err
}
}

if apierrs.IsNotFound(err) || strings.Contains(consoleRoute.Spec.Host, "redhat.com") {
err := deploy.DeployManifestsFromPath(dsciInit, cli,
defaultManifestPath+"/monitoring/blackbox-exporter/internal",
dsciInit.Spec.Monitoring.Namespace, s)
if err != nil {
return err
}

} else {
err := deploy.DeployManifestsFromPath(dsciInit, cli,
defaultManifestPath+"/monitoring/blackbox-exporter/external",
dsciInit.Spec.Monitoring.Namespace, s)
if err != nil {
return err
}
}
return nil
}

func (r *DSCInitializationReconciler) configureManagedMonitoring(dscInit *dsci.DSCInitialization) error {
// configure Alertmanager
if err := configureAlertManager(dscInit, r); err != nil {
fmt.Printf("Error in alertmanager")
return err
}

// configure Prometheus
if err := configurePrometheus(dscInit, r); err != nil {
fmt.Printf("Error in prometheus")
return err
}

// configure Blackbox exporter
if err := configureBlackboxExporter(dscInit, r.Client, r.Scheme); err != nil {
fmt.Printf("Error in blackbox exporter")
return err
}
return nil
}

func createMonitoringProxySecret(name string, dsciInit *dsci.DSCInitialization, cli client.Client, s *runtime.Scheme) error {

sessionSecret, err := GenerateRandomHex(32)
if err != nil {
return err
}

desiredProxySecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: dsciInit.Spec.Monitoring.Namespace,
},
Data: map[string][]byte{
"session_secret": []byte(b64.StdEncoding.EncodeToString(sessionSecret)),
},
}

foundProxySecret := &corev1.Secret{}
err = cli.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: dsciInit.Spec.Monitoring.Namespace}, foundProxySecret)
if err != nil {
if apierrs.IsNotFound(err) {
// Set Controller reference
err = ctrl.SetControllerReference(dsciInit, desiredProxySecret, s)
if err != nil {
return err
}
err = cli.Create(context.TODO(), desiredProxySecret)
if err != nil && !apierrs.IsAlreadyExists(err) {
return err
}
} else {
return err
}
}
return nil

}

func replaceInAlertManagerConfigmap(cli client.Client, dsciInit *dsci.DSCInitialization, cmName, replaceVariable, replaceValue string) error {
prometheusConfig := &corev1.ConfigMap{}
err := cli.Get(context.TODO(), client.ObjectKey{Name: cmName, Namespace: dsciInit.Spec.Monitoring.Namespace}, prometheusConfig)
if err != nil {
if apierrs.IsNotFound(err) {
return nil
}
return err
}
prometheusAlertmanagerContent := prometheusConfig.Data["alertmanager.yml"]
prometheusAlertmanagerContent = strings.ReplaceAll(prometheusAlertmanagerContent, replaceVariable, replaceValue)

prometheusConfig.Data["alertmanager.yml"] = prometheusAlertmanagerContent
return cli.Update(context.TODO(), prometheusConfig)
}

func getMonitoringData(data string) (string, error) {
// Create a new SHA-256 hash object
hash := sha256.New()

// Write the input data to the hash object
_, err := hash.Write([]byte(data))
if err != nil {
return "", err
}

// Get the computed hash sum
hashSum := hash.Sum(nil)

// Encode the hash sum to Base64
encodedData := b64.StdEncoding.EncodeToString(hashSum)

return encodedData, nil
}
35 changes: 35 additions & 0 deletions controllers/dscinitialization/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,25 @@ import (
"k8s.io/client-go/util/retry"
"reflect"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
authv1 "k8s.io/api/rbac/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

dsci "github.com/opendatahub-io/opendatahub-operator/apis/dscinitialization/v1alpha1"
)

var (
resourceInterval = 10 * time.Second
resourceTimeout = 1 * time.Minute
)

// createOdhNamespace creates a Namespace with given name and with ODH defaults. The defaults include:
// - Odh specific labels
// - Pod security labels for baseline permissions
Expand Down Expand Up @@ -58,6 +65,12 @@ func (r *DSCInitializationReconciler) createOdhNamespace(dscInit *dsci.DSCInitia
r.Log.Error(err, "Unable to fetch namespace", "name", name)
return err
}
} else if dscInit.Spec.Monitoring.Enabled {
err = r.Patch(ctx, foundNamespace, client.RawPatch(types.MergePatchType,
[]byte(`{"metadata": {"labels": {"openshift.io/cluster-monitoring": "true"}}}`)))
if err != nil {
return err
}
}

// Create default NetworkPolicy for the namespace
Expand Down Expand Up @@ -202,6 +215,28 @@ func CompareNotebookNetworkPolicies(np1 netv1.NetworkPolicy, np2 netv1.NetworkPo
reflect.DeepEqual(np1.Spec, np2.Spec)
}

func (r *DSCInitializationReconciler) waitForManagedSecret(name, namespace string) (*corev1.Secret, error) {
managedSecret := &corev1.Secret{}
err := wait.Poll(resourceInterval, resourceTimeout, func() (done bool, err error) {

err = r.Client.Get(context.TODO(), client.ObjectKey{
Namespace: namespace,
Name: name,
}, managedSecret)

if err != nil {
if apierrs.IsNotFound(err) {
return false, nil
}
return false, err
} else {
return true, nil
}
})

return managedSecret, err
}

func GenerateRandomHex(length int) ([]byte, error) {
// Calculate the required number of bytes
numBytes := length / 2
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ require (
github.com/onsi/ginkgo/v2 v2.6.0
github.com/onsi/gomega v1.24.1
github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd
github.com/openshift/api v3.9.0+incompatible
github.com/openshift/custom-resource-status v1.1.2
k8s.io/api v0.26.2
k8s.io/apiextensions-apiserver v0.26.1
k8s.io/apimachinery v0.26.2
k8s.io/client-go v0.26.1
sigs.k8s.io/controller-runtime v0.14.4
Expand Down Expand Up @@ -70,7 +72,6 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.26.1 // indirect
k8s.io/component-base v0.26.1 // indirect
k8s.io/klog/v2 v2.90.1 // indirect
k8s.io/kube-openapi v0.0.0-20230109183929-3758b55a6596 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,8 @@ github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd h1:6elrLdOa+BRHJVaHnZAHltufWk0pzPZYF67fX9aFCjU=
github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd/go.mod h1:cDMtOZx741HfmmUMmT09PWM8cOBxEJp3ipUHeHPr8F4=
github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4=
github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down
Loading

0 comments on commit 78c1d61

Please sign in to comment.