Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support metrics scrape #2344

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: support metrics scrape #2344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/tenant_crd.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1143,6 +1143,12 @@ Directs the MinIO Operator to use prometheus operator. +

Tenant scrape configuration will be added to prometheus managed by the prometheus-operator.

|*`prometheusOperatorScrapeMetricsPath`* __string array__
|*Optional* +


The name of the Prometheus instance to scrape metrics from.

|*`serviceAccountName`* __string__
|*Optional* +

Expand Down
4 changes: 4 additions & 0 deletions helm/operator/templates/minio.min.io_tenants.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3682,6 +3682,10 @@ spec:
type: string
prometheusOperator:
type: boolean
prometheusOperatorScrapeMetricsPath:
items:
type: string
type: array
readiness:
properties:
exec:
Expand Down
3 changes: 0 additions & 3 deletions pkg/apis/minio.min.io/v2/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,6 @@ const StatefulSetPrefix = "ss"
// StatefulSetLegacyPrefix by old operators
const StatefulSetLegacyPrefix = "zone"

// MinIOPrometheusPathCluster is the path where MinIO tenant exposes cluster Prometheus metrics
const MinIOPrometheusPathCluster = "/minio/v2/metrics/cluster"

// MinIOPrometheusScrapeInterval defines how frequently to scrape targets.
const MinIOPrometheusScrapeInterval = 30 * time.Second

Expand Down
6 changes: 6 additions & 0 deletions pkg/apis/minio.min.io/v2/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,12 @@ type TenantSpec struct {
PrometheusOperator bool `json:"prometheusOperator,omitempty"`
// *Optional* +
//
// The name of the Prometheus instance to scrape metrics from.
//
// +optional
PrometheusOperatorScrapeMetricsPath []string `json:"prometheusOperatorScrapeMetricsPath,omitempty"`
// *Optional* +
//
// The https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes Service Account] to use for running MinIO pods created as part of the Tenant. +
// +optional
ServiceAccountName string `json:"serviceAccountName,omitempty"`
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

81 changes: 46 additions & 35 deletions pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

49 changes: 29 additions & 20 deletions pkg/controller/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package controller
import (
"context"
"errors"
"reflect"
"strings"

promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
Expand Down Expand Up @@ -161,20 +163,34 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
return err
}
} else {
var scrapeConfigs []configmaps.ScrapeConfig
var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig
err := yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs)
if err != nil {
return err
}
// Check if the scrape config is already present
hasScrapeConfig := false
// get other scrape configs
for _, sc := range scrapeConfigs {
if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() {
hasScrapeConfig = true
break
if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc)
}
}
if !hasScrapeConfig {
exceptedScrapeConfigs = append(exceptedScrapeConfigs, promCfg.ScrapeConfigs...)
updateScrapeConfig := false
if len(scrapeConfigs) != len(exceptedScrapeConfigs) {
updateScrapeConfig = true
} else {
for i := range scrapeConfigs {
if scrapeConfigs[i].JobName != exceptedScrapeConfigs[i].JobName ||
scrapeConfigs[i].MetricsPath != exceptedScrapeConfigs[i].MetricsPath ||
scrapeConfigs[i].Scheme != exceptedScrapeConfigs[i].Scheme ||
!reflect.DeepEqual(scrapeConfigs[i].TLSConfig, exceptedScrapeConfigs[i].TLSConfig) ||
!reflect.DeepEqual(scrapeConfigs[i].StaticConfigs, exceptedScrapeConfigs[i].StaticConfigs) {
updateScrapeConfig = true
break
}
}
}
if updateScrapeConfig {
klog.Infof("Adding MinIO tenant Prometheus scrape config")
scrapeConfigs = append(scrapeConfigs, promCfg.ScrapeConfigs...)
scrapeCfgYaml, err := yaml.Marshal(scrapeConfigs)
Expand Down Expand Up @@ -224,27 +240,20 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min
return err
}

var scrapeConfigs []configmaps.ScrapeConfig
var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig
err = yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs)
if err != nil {
return err
}
// Check if the scrape config is present
hasScrapeConfig := false
scIndex := -1
for i, sc := range scrapeConfigs {
if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() {
hasScrapeConfig = true
scIndex = i
break
for _, sc := range scrapeConfigs {
if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc)
}
}
if hasScrapeConfig {
if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) {
klog.Infof("Deleting MinIO tenant Prometheus scrape config")
// Delete the config
newScrapeConfigs := append(scrapeConfigs[:scIndex], scrapeConfigs[scIndex+1:]...)
// Update the secret
scrapeCfgYaml, err := yaml.Marshal(newScrapeConfigs)
scrapeCfgYaml, err := yaml.Marshal(exceptedScrapeConfigs)
Comment on lines +253 to +256
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reflectDeepEqual will always return false due to the implementation of GetPrometheusConfig. This will trigger multiple unnecessary secret recreation.

Please consider the following option instead:

// add new scrape configs
    exceptedScrapeConfigs = append(exceptedScrapeConfigs, promCfg.ScrapeConfigs...)
    // compare scrape configs excluding bearer token to avoid continuous updates
    updateScrapeConfig := false
    if len(scrapeConfigs) != len(exceptedScrapeConfigs) {
      updateScrapeConfig = true
    } else {
      for i, _ := range scrapeConfigs {
        if scrapeConfigs[i].JobName != exceptedScrapeConfigs[i].JobName ||
          scrapeConfigs[i].MetricsPath != exceptedScrapeConfigs[i].MetricsPath ||
          scrapeConfigs[i].Scheme != exceptedScrapeConfigs[i].Scheme ||
          !reflect.DeepEqual(scrapeConfigs[i].TLSConfig, exceptedScrapeConfigs[i].TLSConfig) ||
          !reflect.DeepEqual(scrapeConfigs[i].StaticConfigs, exceptedScrapeConfigs[i].StaticConfigs) {
          updateScrapeConfig = true
          break
        }
      }
    }
    if updateScrapeConfig {
      klog.Infof("Adding MinIO tenant Prometheus scrape config")
      scrapeConfigs = exceptedScrapeConfigs
      scrapeCfgYaml, err := yaml.Marshal(scrapeConfigs)
      if err != nil {
        return err
      }
      secret.Data[miniov2.PrometheusAddlScrapeConfigKey] = scrapeCfgYaml
      _, err = c.kubeClientSet.CoreV1().Secrets(ns).Update(ctx, secret, metav1.UpdateOptions{})
      if err != nil {
        return err
      }
    }

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please take a look @jiuker

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Found that bear_token is generated by time.Now()

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@allanrogerr PTAL again.

if err != nil {
return err
}
Expand Down
35 changes: 21 additions & 14 deletions pkg/resources/configmaps/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,22 +80,29 @@ func GetPrometheusConfig(t *miniov2.Tenant, accessKey, secretKey string) *Promet
ScrapeInterval: miniov2.MinIOPrometheusScrapeInterval,
EvaluationInterval: 30 * time.Second,
},
ScrapeConfigs: []ScrapeConfig{
{
JobName: t.PrometheusConfigJobName(),
BearerToken: bearerToken,
MetricsPath: miniov2.MinIOPrometheusPathCluster,
Scheme: minioScheme,
TLSConfig: tlsConfig{
CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
},
StaticConfigs: []staticConfig{
{
Targets: []string{minioTargets},
},
ScrapeConfigs: []ScrapeConfig{},
}

if len(t.Spec.PrometheusOperatorScrapeMetricsPath) == 0 {
t.Spec.PrometheusOperatorScrapeMetricsPath = []string{"/minio/v2/metrics/cluster"}
}

for index, scrape := range t.Spec.PrometheusOperatorScrapeMetricsPath {
promConfig.ScrapeConfigs = append(promConfig.ScrapeConfigs, ScrapeConfig{
JobName: fmt.Sprintf("%s-%d", t.PrometheusOperatorAddlConfigJobName(), index),
BearerToken: bearerToken,
MetricsPath: scrape,
Scheme: minioScheme,
TLSConfig: tlsConfig{
CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
},
StaticConfigs: []staticConfig{
{
Targets: []string{minioTargets},
},
},
},
})
}

return promConfig
}
4 changes: 4 additions & 0 deletions resources/base/crds/minio.min.io_tenants.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3682,6 +3682,10 @@ spec:
type: string
prometheusOperator:
type: boolean
prometheusOperatorScrapeMetricsPath:
items:
type: string
type: array
readiness:
properties:
exec:
Expand Down
Loading