Merge pull request #33 from mia-platform/feat/update-prometheus-rules

Update prometheus rules

Author: verdienansein

charts/monitoring/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: monitoring
-version: 2.2.1
+version: 2.2.2
 kubeVersion: ">= 1.20.0-0"
-appVersion: v0.73.2
+appVersion: v0.74.0
 description: "A Kubernetes monitoring stack based on Prometheus Operator"
 type: application
 keywords:

charts/monitoring/templates/default-monitors/alertmanager.yaml

@@ -13,7 +13,17 @@ spec:
       - {{ .Release.Namespace | quote }}
   jobLabel: "app.kubernetes.io/name"
   podMetricsEndpoints:
-    - port: "web"
-      path: "/metrics"
-    - port: "reloader-web"
-      path: "/metrics"
+  - path: /metrics
+    port: web
+    metricRelabelings:
+    - action: keep
+      regex: alertmanager.*
+      sourceLabels:
+      - __name__
+  - path: /metrics
+    port: reloader-web
+    metricRelabelings:
+    - action: keep
+      regex: reloader.*
+      sourceLabels:
+      - __name__

charts/monitoring/templates/default-monitors/dns.yaml

@@ -11,8 +11,19 @@ spec:
       k8s-app: "kube-dns"
   namespaceSelector:
     matchNames:
-      - "kube-system"
+    - kube-system
   jobLabel: "k8s-app"
   podMetricsEndpoints:
-    - port: "metrics"
+  - path: /metrics
+    port: metrics
+    metricRelabelings:
+    - action: keep
+      regex: (coredns.*|skydns.*|kubedns.*)
+      sourceLabels:
+      - __name__
+    - action: drop
+      # deprecated metrics, don't count on them
+      regex: coredns_cache_misses_total
+      sourceLabels:
+      - __name__
 {{- end }}

charts/monitoring/templates/default-monitors/kubelet.yaml

@@ -0,0 +1,103 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "mia-monitoring.kubelet.fullname" . | quote }}
+  labels:
+    {{- include "mia-monitoring.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kubelet
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  jobLabel: "app.kubernetes.io/name"
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    metricRelabelings:
+    - action: drop
+      regex: kubelet_(pod_worker_latency_microseconds|pod_start_latency_microseconds|cgroup_manager_latency_microseconds|pod_worker_start_latency_microseconds|pleg_relist_latency_microseconds|pleg_relist_interval_microseconds|runtime_operations|runtime_operations_latency_microseconds|runtime_operations_errors|eviction_stats_age_microseconds|device_plugin_registration_count|device_plugin_alloc_latency_microseconds|network_plugin_operations_latency_microseconds)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: scheduler_(e2e_scheduling_latency_microseconds|scheduling_algorithm_predicate_evaluation|scheduling_algorithm_priority_evaluation|scheduling_algorithm_preemption_evaluation|scheduling_algorithm_latency_microseconds|binding_latency_microseconds|scheduling_latency_seconds)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: apiserver_(request_count|request_latencies|request_latencies_summary|dropped_requests|storage_data_key_generation_latencies_microseconds|storage_transformation_failures_total|storage_transformation_latencies_microseconds|proxy_tunnel_sync_latency_secs|longrunning_gauge|registered_watchers|storage_db_total_size_in_bytes)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: kubelet_docker_(operations|operations_latency_microseconds|operations_errors|operations_timeout)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: reflector_(items_per_list|items_per_watch|list_duration_seconds|lists_total|short_watches_total|watch_duration_seconds|watches_total)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: etcd_(helper_cache_hit_count|helper_cache_miss_count|helper_cache_entry_count|object_counts|request_cache_get_latencies_summary|request_cache_add_latencies_summary|request_latencies_summary)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: transformation_(transformation_latencies_microseconds|failures_total)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: (admission_quota_controller_adds|admission_quota_controller_depth|admission_quota_controller_longest_running_processor_microseconds|admission_quota_controller_queue_latency|admission_quota_controller_unfinished_work_seconds|admission_quota_controller_work_duration|APIServiceOpenAPIAggregationControllerQueue1_adds|APIServiceOpenAPIAggregationControllerQueue1_depth|APIServiceOpenAPIAggregationControllerQueue1_longest_running_processor_microseconds|APIServiceOpenAPIAggregationControllerQueue1_queue_latency|APIServiceOpenAPIAggregationControllerQueue1_retries|APIServiceOpenAPIAggregationControllerQueue1_unfinished_work_seconds|APIServiceOpenAPIAggregationControllerQueue1_work_duration|APIServiceRegistrationController_adds|APIServiceRegistrationController_depth|APIServiceRegistrationController_longest_running_processor_microseconds|APIServiceRegistrationController_queue_latency|APIServiceRegistrationController_retries|APIServiceRegistrationController_unfinished_work_seconds|APIServiceRegistrationController_work_duration|autoregister_adds|autoregister_depth|autoregister_longest_running_processor_microseconds|autoregister_queue_latency|autoregister_retries|autoregister_unfinished_work_seconds|autoregister_work_duration|AvailableConditionController_adds|AvailableConditionController_depth|AvailableConditionController_longest_running_processor_microseconds|AvailableConditionController_queue_latency|AvailableConditionController_retries|AvailableConditionController_unfinished_work_seconds|AvailableConditionController_work_duration|crd_autoregistration_controller_adds|crd_autoregistration_controller_depth|crd_autoregistration_controller_longest_running_processor_microseconds|crd_autoregistration_controller_queue_latency|crd_autoregistration_controller_retries|crd_autoregistration_controller_unfinished_work_seconds|crd_autoregistration_controller_work_duration|crdEstablishing_adds|crdEstablishing_depth|crdEstablishing_longest_running_processor_microseconds|crdEstablishing_queue_latency|crdEstablishing_retries|crdEstablishing_unfinished_work_seconds|crdEstablishing_work_duration|crd_finalizer_adds|crd_finalizer_depth|crd_finalizer_longest_running_processor_microseconds|crd_finalizer_queue_latency|crd_finalizer_retries|crd_finalizer_unfinished_work_seconds|crd_finalizer_work_duration|crd_naming_condition_controller_adds|crd_naming_condition_controller_depth|crd_naming_condition_controller_longest_running_processor_microseconds|crd_naming_condition_controller_queue_latency|crd_naming_condition_controller_retries|crd_naming_condition_controller_unfinished_work_seconds|crd_naming_condition_controller_work_duration|crd_openapi_controller_adds|crd_openapi_controller_depth|crd_openapi_controller_longest_running_processor_microseconds|crd_openapi_controller_queue_latency|crd_openapi_controller_retries|crd_openapi_controller_unfinished_work_seconds|crd_openapi_controller_work_duration|DiscoveryController_adds|DiscoveryController_depth|DiscoveryController_longest_running_processor_microseconds|DiscoveryController_queue_latency|DiscoveryController_retries|DiscoveryController_unfinished_work_seconds|DiscoveryController_work_duration|kubeproxy_sync_proxy_rules_latency_microseconds|non_structural_schema_condition_controller_adds|non_structural_schema_condition_controller_depth|non_structural_schema_condition_controller_longest_running_processor_microseconds|non_structural_schema_condition_controller_queue_latency|non_structural_schema_condition_controller_retries|non_structural_schema_condition_controller_unfinished_work_seconds|non_structural_schema_condition_controller_work_duration|rest_client_request_latency_seconds|storage_operation_errors_total|storage_operation_status_count)
+      sourceLabels:
+      - __name__
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    honorTimestamps: false
+    interval: 30s
+    metricRelabelings:
+    - action: drop
+      regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: (container_spec_.*|container_file_descriptors|container_sockets|container_threads_max|container_threads|container_start_time_seconds|container_last_seen);;
+      sourceLabels:
+      - __name__
+      - pod
+      - namespace
+    - action: drop
+      regex: (container_blkio_device_usage_total);.+
+      sourceLabels:
+      - __name__
+      - container
+    path: /metrics/cadvisor
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    path: /metrics/probes
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true

charts/monitoring/templates/default-monitors/kubestatemerics.yaml

@@ -13,7 +13,27 @@ spec:
       - {{ .Release.Namespace | quote }}
   jobLabel: "app.kubernetes.io/name"
   podMetricsEndpoints:
-    - port: "web"
-      honorLabels: true
-    - port: "metrics"
-      honorLabels: true
+  - path: /metrics
+    port: self-metrics
+    honorLabels: true
+    interval: 30s
+    metricRelabelings:
+    - action: keep
+      regex: kube.*
+      sourceLabels:
+      - __name__
+  - path: /metrics
+    port: k8s-metrics
+    interval: 30s
+    relabelings:
+    - action: labeldrop
+      regex: (pod|service|endpoint|namespace)
+    metricRelabelings:
+    - action: drop
+      regex: (kube_endpoint_address_not_ready|kube_endpoint_address_available)
+      sourceLabels:
+      - __name__
+    - action: keep
+      regex: kube.*
+      sourceLabels:
+      - __name__

charts/monitoring/templates/default-monitors/nodeexporter.yaml

@@ -13,5 +13,17 @@ spec:
       - {{ .Release.Namespace | quote }}
   jobLabel: "app.kubernetes.io/name"
   podMetricsEndpoints:
-    - port: "metrics"
-      scrapeTimeout: "10s"
+  - path: /metrics
+    port: metrics
+    relabelings:
+    - action: replace
+      regex: (.*)
+      replacement: $1
+      sourceLabels:
+      - __meta_kubernetes_pod_node_name
+      targetLabel: instance
+    metricRelabelings:
+    - action: keep
+      regex: node.*
+      sourceLabels:
+      - __name__

charts/monitoring/templates/default-monitors/operator.yaml

@@ -13,13 +13,18 @@ spec:
       - {{ .Release.Namespace | quote }}
   jobLabel: "app.kubernetes.io/name"
   endpoints:
-    - port: "websecure"
-      scheme: "https"
-      tlsConfig:
-        serverName: {{ include "mia-monitoring.fullname" . | quote }}
-        ca:
-          secret:
-            name: {{ include "mia-monitoring.tlsSecretName" . | quote }}
-            key: ca
-            optional: false
-      honorLabels: true
+  - port: websecure
+    path: /metrics
+    scheme: https
+    tlsConfig:
+      serverName: {{ include "mia-monitoring.fullname" . | quote }}
+      ca:
+        secret:
+          name: {{ include "mia-monitoring.tlsSecretName" . | quote }}
+          key: ca
+          optional: false
+    metricRelabelings:
+    - action: keep
+      regex: prometheus_operator.*
+      sourceLabels:
+      - __name__

charts/monitoring/templates/default-monitors/prometheus.yaml

@@ -13,7 +13,17 @@ spec:
       - {{ .Release.Namespace | quote }}
   jobLabel: "app.kubernetes.io/name"
   podMetricsEndpoints:
-    - port: "web"
-      path: "/metrics"
-    - port: "reloader-web"
-      path: "/metrics"
+  - path: /metrics
+    port: web
+    metricRelabelings:
+    - action: keep
+      regex: prometheus.*
+      sourceLabels:
+      - __name__
+  - path: /metrics
+    port: reloader-web
+    metricRelabelings:
+    - action: keep
+      regex: reloader.*
+      sourceLabels:
+      - __name__