Replace KSM and helm-exporter with built-in metrics (azimuth-cloud#462)

.github/workflows/update-dependencies.yaml

@@ -91,13 +91,6 @@ jobs:
             chart_repo_jsonpath: dependencies[0].repository
             chart_version_jsonpath: dependencies[0].version
 
-          - key: kube-state-metrics
-            component: metrics
-            path: ./charts/server/Chart.yaml
-            chart_name_jsonpath: dependencies[0].name
-            chart_repo_jsonpath: dependencies[0].repository
-            chart_version_jsonpath: dependencies[0].version
-
     name: ${{ matrix.key }}
     steps:
       - name: Checkout

charts/server/Chart.yaml

@@ -6,11 +6,3 @@ type: application
 # The version and appVersion are updated by the chart build script
 version: 0.1.0
 appVersion: main
-
-dependencies:
-  # prometheus-community/kube-state-metrics to produce metrics on clusters
-  - name: kube-state-metrics
-    repository: https://prometheus-community.github.io/helm-charts
-    version: 5.19.1
-    alias: metrics
-    condition: metrics.enabled

charts/server/templates/sync/deployment.yaml

@@ -31,6 +31,10 @@ spec:
           securityContext: {{ toYaml .Values.sync.securityContext | nindent 12 }}
           image: {{ printf "%s:%s" .Values.sync.image.repository (default .Chart.AppVersion .Values.sync.image.tag) }}
           imagePullPolicy: {{ .Values.sync.image.pullPolicy }}
+          ports:
+            - name: metrics
+              containerPort: 8080
+              protocol: TCP
           # Tell the sync process which namespace it is running in
           env:
             - name: ZENITH_SYNC__KUBERNETES__SELF_NAMESPACE

charts/server/templates/metrics/prometheusrule.yaml → charts/server/templates/sync/prometheusrule.yaml

@@ -1,9 +1,14 @@
-{{- if and .Values.metrics.enabled .Values.metrics.prometheus.rules.enabled }}
+{{-
+  if and
+    .Values.sync.enabled
+    .Values.metrics.enabled
+    .Values.metrics.prometheus.rules.enabled
+}}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: {{ include "zenith.componentname" (list . "metrics") }}
-  labels: {{ include "zenith.componentLabels" (list . "metrics") | nindent 4 }}
+  name: {{ include "zenith.componentname" (list . "sync") }}
+  labels: {{ include "zenith.componentLabels" (list . "sync") | nindent 4 }}
 spec:
   groups:
     - name: zenith.recording-rules
@@ -18,11 +23,6 @@ spec:
           expr: >-
             sum(topk(1, label_replace(zenith_service_endpoint_info, "status", "$1", "endpoint_status", "(.*)")) by(service_name, endpoint_id, status)) by(service_name, status)
 
-        # This metric indicates the status of the Helm release
-        - record: zenith_service_helm_status
-          expr: >-
-            count(topk(1, label_replace(helm_chart_info, "service_name", "$1", "release", "(.+)")) by(service_name, status)) by(service_name, status)
-
     # Zenith service alerts
     - name: zenith.alerts
       rules:

charts/server/templates/sync/service.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.sync.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "zenith.componentname" (list . "sync") }}
+  labels: {{ include "zenith.componentLabels" (list . "sync") | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: metrics
+      port: 8080
+      targetPort: metrics
+      protocol: TCP
+  selector: {{ include "zenith.componentSelectorLabels" (list . "sync") | nindent 4 }}
+{{- end }}

charts/server/templates/sync/servicemonitor.yaml

@@ -0,0 +1,19 @@
+{{-
+  if and
+    .Values.sync.enabled
+    .Values.metrics.enabled
+    .Values.metrics.prometheus.monitor.enabled
+}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "zenith.componentname" (list . "sync") }}
+  labels: {{ include "zenith.componentLabels" (list . "sync") | nindent 4 }}
+spec:
+  endpoints:
+    - honorLabels: true
+      port: metrics
+  jobLabel: app.kubernetes.io/name
+  selector:
+    matchLabels: {{ include "zenith.componentSelectorLabels" (list . "sync") | nindent 6 }}
+{{- end }}

charts/server/values.yaml

@@ -156,99 +156,3 @@ metrics:
       enabled: true
     monitor:
       enabled: true
-      honorLabels: true
-  # Disable all the default collectors
-  collectors: []
-  # Allow kube-state-metrics read-only access to our CRDs
-  rbac:
-    create: true
-    extraRules:
-      - apiGroups:
-          - zenith.stackhpc.com
-        resources:
-          - endpoints
-          - leases
-          - services
-        verbs:
-          - list
-          - watch
-          - get
-  # Configure kube-state-metrics to report only on our custom resources
-  extraArgs:
-    - --custom-resource-state-only=true
-  customResourceState:
-    enabled: true
-    config:
-      kind: CustomResourceStateMetrics
-      spec:
-        resources:
-          - groupVersionKind:
-              group: zenith.stackhpc.com
-              version: v1alpha1
-              kind: Service
-            metricNamePrefix: zenith_service
-            labelsFromPath:
-              service_namespace: [metadata, namespace]
-              service_name: [metadata, name]
-            metrics:
-              - name: info
-                help: "Service info"
-                each:
-                  type: Info
-                  info:
-                    labelsFromPath:
-                      created_at: [metadata, creationTimestamp]
-                      fingerprint: [spec, publicKeyFingerprint]
-
-          - groupVersionKind:
-              group: zenith.stackhpc.com
-              version: v1alpha1
-              kind: Endpoints
-            metricNamePrefix: zenith_service
-            labelsFromPath:
-              service_namespace: [metadata, namespace]
-              service_name: [metadata, name]
-            metrics:
-              - name: endpoint_info
-                help: "Metric for the endpoints of a service"
-                each:
-                  type: Info
-                  info:
-                    path: [spec, endpoints]
-                    labelFromKey: endpoint_id
-                    labelsFromPath:
-                      endpoint_address: [address]
-                      endpoint_port: [port]
-                      endpoint_status: [status]
-
-  # Configuration for the Helm exporter
-  helmExporter:
-    # The image to use for the Helm exporter
-    # We use a synced version to avoid a dependency on Docker Hub
-    image:
-      repository: ghcr.io/stackhpc/zenith/helm-exporter
-      pullPolicy: IfNotPresent
-      tag: 1.2.16
-    imagePullSecrets: []
-    # Customise pod-level security context for helm exporter pods
-    podSecurityContext:
-      runAsNonRoot: true
-      runAsUser: 1000
-      runAsGroup: 1000
-      fsGroup: 1000
-    # Customise container-level security context for helm exporter pods
-    securityContext:
-      allowPrivilegeEscalation: false
-      capabilities:
-        drop: [ALL]
-      readOnlyRootFilesystem: true
-    # Resources for helm exporter containers
-    resources: {}
-    # Customise annotations for helm exporter pods
-    podAnnotations: {}
-    # Customise node selector for helm exporter pods
-    nodeSelector: {}
-    # Customise tolerations for helm exporter pods
-    tolerations: []
-    # Customise affinity rules for helm exporter pods
-    affinity: {}

sync/requirements.txt

@@ -1,19 +1,26 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
 annotated-types==0.7.0
 anyio==4.4.0
+async-timeout==4.0.3
+attrs==23.2.0
 certifi==2024.6.2
 click==8.1.7
 configomatic==0.2.0
 easykube==0.3.2
 easysemver==0.1.0
 exceptiongroup==1.2.1
+frozenlist==1.4.1
 h11==0.14.0
 httpcore==1.0.5
 httpx==0.27.0
 idna==3.7
 kube-custom-resource==0.3.0
+multidict==6.0.5
 pydantic==2.7.3
 pydantic_core==2.18.4
 pyhelm3==0.3.3
 PyYAML==6.0.1
 sniffio==1.3.1
 typing_extensions==4.12.1
+yarl==1.9.4

sync/setup.cfg

@@ -11,6 +11,7 @@ zip_safe = False
 include_package_data = True
 packages = find_namespace:
 install_requires =
+    aiohttp
     click
     configomatic[yaml]
     easykube