diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index 678d205bf69..22146a125d6 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -1143,6 +1143,12 @@ Directs the MinIO Operator to use prometheus operator. + Tenant scrape configuration will be added to prometheus managed by the prometheus-operator. +|*`prometheusOperatorScrapeMetricsPath`* __string array__ +|*Optional* + + + +The name of the Prometheus instance to scrape metrics from. + |*`serviceAccountName`* __string__ |*Optional* + diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index 72d7e591fc6..d28320431ee 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -3682,6 +3682,10 @@ spec: type: string prometheusOperator: type: boolean + prometheusOperatorScrapeMetricsPath: + items: + type: string + type: array readiness: properties: exec: diff --git a/pkg/apis/minio.min.io/v2/constants.go b/pkg/apis/minio.min.io/v2/constants.go index ad4354679be..61b628e4315 100644 --- a/pkg/apis/minio.min.io/v2/constants.go +++ b/pkg/apis/minio.min.io/v2/constants.go @@ -197,9 +197,6 @@ const StatefulSetPrefix = "ss" // StatefulSetLegacyPrefix by old operators const StatefulSetLegacyPrefix = "zone" -// MinIOPrometheusPathCluster is the path where MinIO tenant exposes cluster Prometheus metrics -const MinIOPrometheusPathCluster = "/minio/v2/metrics/cluster" - // MinIOPrometheusScrapeInterval defines how frequently to scrape targets. const MinIOPrometheusScrapeInterval = 30 * time.Second diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 115a390ff21..e3a3b6467c4 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -278,6 +278,12 @@ type TenantSpec struct { PrometheusOperator bool `json:"prometheusOperator,omitempty"` // *Optional* + // + // The name of the Prometheus instance to scrape metrics from. + // + // +optional + PrometheusOperatorScrapeMetricsPath []string `json:"prometheusOperatorScrapeMetricsPath,omitempty"` + // *Optional* + + // // The https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes Service Account] to use for running MinIO pods created as part of the Tenant. + // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` diff --git a/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go b/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go index cec7292ab06..e181795aa6a 100644 --- a/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go +++ b/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go @@ -771,6 +771,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { *out = new(KESConfig) (*in).DeepCopyInto(*out) } + if in.PrometheusOperatorScrapeMetricsPath != nil { + in, out := &in.PrometheusOperatorScrapeMetricsPath, &out.PrometheusOperatorScrapeMetricsPath + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.SideCars != nil { in, out := &in.SideCars, &out.SideCars *out = new(SideCars) diff --git a/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go b/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go index 74acd245bd0..e241708a104 100644 --- a/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go +++ b/pkg/client/applyconfiguration/minio.min.io/v2/tenantspec.go @@ -27,41 +27,42 @@ import ( // TenantSpecApplyConfiguration represents a declarative configuration of the TenantSpec type for use // with apply. type TenantSpecApplyConfiguration struct { - Pools []PoolApplyConfiguration `json:"pools,omitempty"` - Image *string `json:"image,omitempty"` - ImagePullSecret *v1.LocalObjectReference `json:"imagePullSecret,omitempty"` - PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` - Env []v1.EnvVar `json:"env,omitempty"` - ExternalCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCertSecret,omitempty"` - ExternalCaCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCaCertSecret,omitempty"` - ExternalClientCertSecret *LocalCertificateReferenceApplyConfiguration `json:"externalClientCertSecret,omitempty"` - ExternalClientCertSecrets []*miniominiov2.LocalCertificateReference `json:"externalClientCertSecrets,omitempty"` - Mountpath *string `json:"mountPath,omitempty"` - Subpath *string `json:"subPath,omitempty"` - RequestAutoCert *bool `json:"requestAutoCert,omitempty"` - CertExpiryAlertThreshold *int32 `json:"certExpiryAlertThreshold,omitempty"` - Liveness *v1.Probe `json:"liveness,omitempty"` - Readiness *v1.Probe `json:"readiness,omitempty"` - Startup *v1.Probe `json:"startup,omitempty"` - Lifecycle *v1.Lifecycle `json:"lifecycle,omitempty"` - Features *FeaturesApplyConfiguration `json:"features,omitempty"` - CertConfig *CertificateConfigApplyConfiguration `json:"certConfig,omitempty"` - KES *KESConfigApplyConfiguration `json:"kes,omitempty"` - PrometheusOperator *bool `json:"prometheusOperator,omitempty"` - ServiceAccountName *string `json:"serviceAccountName,omitempty"` - PriorityClassName *string `json:"priorityClassName,omitempty"` - ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` - SideCars *SideCarsApplyConfiguration `json:"sideCars,omitempty"` - ExposeServices *ExposeServicesApplyConfiguration `json:"exposeServices,omitempty"` - ServiceMetadata *ServiceMetadataApplyConfiguration `json:"serviceMetadata,omitempty"` - PoolsMetadata *PoolsMetadataApplyConfiguration `json:"poolsMetadata,omitempty"` - Users []v1.LocalObjectReference `json:"users,omitempty"` - Buckets []BucketApplyConfiguration `json:"buckets,omitempty"` - Logging *LoggingApplyConfiguration `json:"logging,omitempty"` - Configuration *v1.LocalObjectReference `json:"configuration,omitempty"` - InitContainers []v1.Container `json:"initContainers,omitempty"` - AdditionalVolumes []v1.Volume `json:"additionalVolumes,omitempty"` - AdditionalVolumeMounts []v1.VolumeMount `json:"additionalVolumeMounts,omitempty"` + Pools []PoolApplyConfiguration `json:"pools,omitempty"` + Image *string `json:"image,omitempty"` + ImagePullSecret *v1.LocalObjectReference `json:"imagePullSecret,omitempty"` + PodManagementPolicy *appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` + Env []v1.EnvVar `json:"env,omitempty"` + ExternalCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCertSecret,omitempty"` + ExternalCaCertSecret []*miniominiov2.LocalCertificateReference `json:"externalCaCertSecret,omitempty"` + ExternalClientCertSecret *LocalCertificateReferenceApplyConfiguration `json:"externalClientCertSecret,omitempty"` + ExternalClientCertSecrets []*miniominiov2.LocalCertificateReference `json:"externalClientCertSecrets,omitempty"` + Mountpath *string `json:"mountPath,omitempty"` + Subpath *string `json:"subPath,omitempty"` + RequestAutoCert *bool `json:"requestAutoCert,omitempty"` + CertExpiryAlertThreshold *int32 `json:"certExpiryAlertThreshold,omitempty"` + Liveness *v1.Probe `json:"liveness,omitempty"` + Readiness *v1.Probe `json:"readiness,omitempty"` + Startup *v1.Probe `json:"startup,omitempty"` + Lifecycle *v1.Lifecycle `json:"lifecycle,omitempty"` + Features *FeaturesApplyConfiguration `json:"features,omitempty"` + CertConfig *CertificateConfigApplyConfiguration `json:"certConfig,omitempty"` + KES *KESConfigApplyConfiguration `json:"kes,omitempty"` + PrometheusOperator *bool `json:"prometheusOperator,omitempty"` + PrometheusOperatorScrapeMetricsPath []string `json:"prometheusOperatorScrapeMetricsPath,omitempty"` + ServiceAccountName *string `json:"serviceAccountName,omitempty"` + PriorityClassName *string `json:"priorityClassName,omitempty"` + ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` + SideCars *SideCarsApplyConfiguration `json:"sideCars,omitempty"` + ExposeServices *ExposeServicesApplyConfiguration `json:"exposeServices,omitempty"` + ServiceMetadata *ServiceMetadataApplyConfiguration `json:"serviceMetadata,omitempty"` + PoolsMetadata *PoolsMetadataApplyConfiguration `json:"poolsMetadata,omitempty"` + Users []v1.LocalObjectReference `json:"users,omitempty"` + Buckets []BucketApplyConfiguration `json:"buckets,omitempty"` + Logging *LoggingApplyConfiguration `json:"logging,omitempty"` + Configuration *v1.LocalObjectReference `json:"configuration,omitempty"` + InitContainers []v1.Container `json:"initContainers,omitempty"` + AdditionalVolumes []v1.Volume `json:"additionalVolumes,omitempty"` + AdditionalVolumeMounts []v1.VolumeMount `json:"additionalVolumeMounts,omitempty"` } // TenantSpecApplyConfiguration constructs a declarative configuration of the TenantSpec type for use with @@ -260,6 +261,16 @@ func (b *TenantSpecApplyConfiguration) WithPrometheusOperator(value bool) *Tenan return b } +// WithPrometheusOperatorScrapeMetricsPath adds the given value to the PrometheusOperatorScrapeMetricsPath field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the PrometheusOperatorScrapeMetricsPath field. +func (b *TenantSpecApplyConfiguration) WithPrometheusOperatorScrapeMetricsPath(values ...string) *TenantSpecApplyConfiguration { + for i := range values { + b.PrometheusOperatorScrapeMetricsPath = append(b.PrometheusOperatorScrapeMetricsPath, values[i]) + } + return b +} + // WithServiceAccountName sets the ServiceAccountName field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the ServiceAccountName field is set to the value of the last call. diff --git a/pkg/controller/prometheus.go b/pkg/controller/prometheus.go index 1b231992d84..c5d72bceaec 100644 --- a/pkg/controller/prometheus.go +++ b/pkg/controller/prometheus.go @@ -17,6 +17,8 @@ package controller import ( "context" "errors" + "reflect" + "strings" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" promv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" @@ -161,20 +163,20 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten return err } } else { - var scrapeConfigs []configmaps.ScrapeConfig + var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig err := yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs) if err != nil { return err } - // Check if the scrape config is already present - hasScrapeConfig := false + // get other scrape configs for _, sc := range scrapeConfigs { - if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() { - hasScrapeConfig = true - break + if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) { + exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } - if !hasScrapeConfig { + // add new scrape configs + exceptedScrapeConfigs = append(exceptedScrapeConfigs, promCfg.ScrapeConfigs...) + if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) { klog.Infof("Adding MinIO tenant Prometheus scrape config") scrapeConfigs = append(scrapeConfigs, promCfg.ScrapeConfigs...) scrapeCfgYaml, err := yaml.Marshal(scrapeConfigs) @@ -224,27 +226,20 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min return err } - var scrapeConfigs []configmaps.ScrapeConfig + var scrapeConfigs, exceptedScrapeConfigs []configmaps.ScrapeConfig err = yaml.Unmarshal(secret.Data[miniov2.PrometheusAddlScrapeConfigKey], &scrapeConfigs) if err != nil { return err } - // Check if the scrape config is present - hasScrapeConfig := false - scIndex := -1 - for i, sc := range scrapeConfigs { - if sc.JobName == tenant.PrometheusOperatorAddlConfigJobName() { - hasScrapeConfig = true - scIndex = i - break + for _, sc := range scrapeConfigs { + if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) { + exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc) } } - if hasScrapeConfig { + if !reflect.DeepEqual(scrapeConfigs, exceptedScrapeConfigs) { klog.Infof("Deleting MinIO tenant Prometheus scrape config") - // Delete the config - newScrapeConfigs := append(scrapeConfigs[:scIndex], scrapeConfigs[scIndex+1:]...) // Update the secret - scrapeCfgYaml, err := yaml.Marshal(newScrapeConfigs) + scrapeCfgYaml, err := yaml.Marshal(exceptedScrapeConfigs) if err != nil { return err } diff --git a/pkg/resources/configmaps/prometheus.go b/pkg/resources/configmaps/prometheus.go index 82b1c5c6171..20aed0bba80 100644 --- a/pkg/resources/configmaps/prometheus.go +++ b/pkg/resources/configmaps/prometheus.go @@ -80,22 +80,29 @@ func GetPrometheusConfig(t *miniov2.Tenant, accessKey, secretKey string) *Promet ScrapeInterval: miniov2.MinIOPrometheusScrapeInterval, EvaluationInterval: 30 * time.Second, }, - ScrapeConfigs: []ScrapeConfig{ - { - JobName: t.PrometheusConfigJobName(), - BearerToken: bearerToken, - MetricsPath: miniov2.MinIOPrometheusPathCluster, - Scheme: minioScheme, - TLSConfig: tlsConfig{ - CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", - }, - StaticConfigs: []staticConfig{ - { - Targets: []string{minioTargets}, - }, + ScrapeConfigs: []ScrapeConfig{}, + } + + if len(t.Spec.PrometheusOperatorScrapeMetricsPath) == 0 { + t.Spec.PrometheusOperatorScrapeMetricsPath = []string{"/minio/v2/metrics/cluster"} + } + + for index, scrape := range t.Spec.PrometheusOperatorScrapeMetricsPath { + promConfig.ScrapeConfigs = append(promConfig.ScrapeConfigs, ScrapeConfig{ + JobName: fmt.Sprintf("%s-%d", t.PrometheusOperatorAddlConfigJobName(), index), + BearerToken: bearerToken, + MetricsPath: scrape, + Scheme: minioScheme, + TLSConfig: tlsConfig{ + CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + }, + StaticConfigs: []staticConfig{ + { + Targets: []string{minioTargets}, }, }, - }, + }) } + return promConfig } diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index 72d7e591fc6..d28320431ee 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -3682,6 +3682,10 @@ spec: type: string prometheusOperator: type: boolean + prometheusOperatorScrapeMetricsPath: + items: + type: string + type: array readiness: properties: exec: