Merge pull request #3 from paradeum-team/jyliu

add prometheus rules, update README.md

charts/ipfs-cluster/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: 1.0.2
 description: IPFS Cluster chart
 name: ipfs-cluster
-version: 0.0.10
+version: 0.0.11
 home: https://ipfscluster.io/
 keywords:
   - ipfs

charts/ipfs-cluster/README.md

@@ -2,7 +2,7 @@
 
 IPFS Cluster chart
 
-![Version: 0.0.9](https://img.shields.io/badge/Version-0.0.9-informational?style=flat-square)
+![Version: 0.0.11](https://img.shields.io/badge/Version-0.0.11-informational?style=flat-square)
 
 ## Installing the Chart
 
@@ -15,7 +15,7 @@ helm repo add paradeum-team https://paradeum-team.github.io/helm-charts/
 ### Install chart
 
 ```
-helm install my-ipfs-cluster paradeum-team/ipfs-cluster --version 0.0.9
+helm install my-ipfs-cluster paradeum-team/ipfs-cluster --version 0.0.11
 ```
 
 ## Values
@@ -36,13 +36,16 @@ helm install my-ipfs-cluster paradeum-team/ipfs-cluster --version 0.0.9
 | ipfsClusterImage.tag | string | `"1.0.2"` |  |
 | ipfsImage.repo | string | `"ipfs/go-ipfs"` |  |
 | ipfsImage.tag | string | `"v0.13.1"` |  |
+| monitor.alert.pinsPinErrorThreshold | int | `10` |  |
+| monitor.alert.pinsPinQueuedThreshold | int | `10` |  |
+| monitor.alert.pinsPinningThreshold | int | `10` |  |
+| monitor.app | string | `"kube-prometheus-stack"` |  |
+| monitor.enabled | bool | `false` |  |
+| monitor.release | string | `"prometheus-community"` |  |
 | nodeSelector | object | `{}` |  |
 | persistence.clusterStorage | string | `"5Gi"` |  |
 | persistence.enabled | bool | `true` |  |
 | persistence.ipfsStorage | string | `"200Gi"` |  |
-| podMonitor.enabled | bool | `false` |  |
-| podMonitor.monitor.app | string | `"kube-prometheus-stack"` |  |
-| podMonitor.monitor.release | string | `"prometheus-community"` |  |
 | replicaCount | int | `1` |  |
 | serviceExternalIPs | list | `[]` |  |
 | tolerations | list | `[]` |  |

charts/ipfs-cluster/templates/pod-monitor.yaml

@@ -1,5 +1,5 @@
-{{- if $.Values.podMonitor }}
-{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.podMonitor.enabled ) }}
+{{- if $.Values.monitor }}
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.monitor.enabled ) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -9,7 +9,7 @@ metadata:
   labels:
       app: {{ .Release.Name }}
       component: ipfs-cluster
-      release: {{ .Values.podMonitor.monitor.release }}
+      release: {{ .Values.monitor.release }}
 spec:
   selector:
     matchLabels:

charts/ipfs-cluster/templates/rules.yaml

@@ -0,0 +1,37 @@
+{{- if .Values.monitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .Release.Name }}-rules
+  labels:
+      app: {{ .Values.monitor.app }}
+      app.kubernetes.io/name: {{ .Release.Name }}
+      component: ipfs-cluster
+      release: {{ .Values.monitor.release }}
+spec:
+  groups:
+    - name: {{ .Release.Name }}.rules
+      rules:
+      - alert: IpfsClusterPinsPinError
+        annotations:
+          message: 'Ipfs cluster pins pin error  (pod {{`{{`}} $labels.pod {{`}}`}}, job {{`{{`}} $labels.job {{`}}`}}) > {{ .Values.monitor.alert.pinsPinErrorThreshold }}, VALUE = {{`{{`}} $value {{`}}`}}'
+        expr: max by(job, pod)(ipfscluster_pins_pin_error) > {{ .Values.monitor.alert.pinsPinErrorThreshold }}
+        for: 5m
+        labels:
+          severity: warning
+      - alert: IpfsClusterPinsPinQueuedTooLarge
+        annotations:
+          message: 'Ipfs cluster pins pin queued  (pod {{`{{`}} $labels.pod {{`}}`}}, job {{`{{`}} $labels.job {{`}}`}}) > {{ .Values.monitor.alert.pinsPinQueuedThreshold }}, VALUE = {{`{{`}} $value {{`}}`}}'
+        expr: max by(job, pod)(ipfscluster_pins_pin_queued) > {{ .Values.monitor.alert.pinsPinQueuedThreshold }}
+        for: 5m
+        labels:
+          severity: warning
+      - alert: IpfsClusterPinsPinningTooLarge
+        annotations:
+          message: 'Ipfs cluster pins pin queued  (pod {{`{{`}} $labels.pod {{`}}`}}, job {{`{{`}} $labels.job {{`}}`}}) > {{ .Values.monitor.alert.pinsPinningThreshold }}, VALUE = {{`{{`}} $value {{`}}`}}'
+        expr: max by(job, instance)(ipfscluster_pins_pinning) > {{ .Values.monitor.alert.pinsPinningThreshold }}
+        for: 5m
+        labels:
+          severity: warning
+{{- end }}
+

charts/ipfs-cluster/templates/secret.yaml

@@ -5,4 +5,4 @@ metadata:
 type: Opaque
 data:
   cluster-secret: {{ .Values.clusterSecret | b64enc }}
-  bootstrap-peer-priv-key: {{ .Values.bootstrapPeerPrivateKey | b64enc }}
+  bootstrap-peer-priv-key: {{ .Values.bootstrapPeerPrivateKey | b64enc }}

charts/ipfs-cluster/templates/service.yaml

@@ -63,4 +63,4 @@ spec:
     targetPort: api-http
     port: 9094    
   selector:
-    app: {{ .Release.Name }}
+    app: {{ .Release.Name }}      

charts/ipfs-cluster/templates/statefulset.yaml

@@ -114,12 +114,14 @@ spec:
               value: "{{ .Values.clusterCRDTtrustedPeers }}"
             - name: SVC_NAME
               value: {{ include "ipfs-cluster.serviceName" . }}
+            {{- if .Values.monitor.enabled }}
             - name: CLUSTER_METRICS_ENABLESTATS
-              value: "{{ .Values.podMonitor.enabled }}"
+              value: "{{ .Values.monitor.enabled }}"
             - name: CLUSTER_METRICS_PROMETHEUSENDPOINT
               value: "/ip4/0.0.0.0/tcp/8888/"
             - name: CLUSTER_METRICS_REPORTING_INTERVAL
               value: "15s"
+            {{- end }}
             - name: CLUSTER_LEAVEONSHUTDOWN
               value: "true"
           ports:

charts/ipfs-cluster/values.yaml

@@ -36,12 +36,16 @@ persistence:
   clusterStorage: 5Gi
   ipfsStorage: 200Gi
 
-podMonitor:
+monitor:
   enabled: false
-  monitor:
-    app: kube-prometheus-stack
-    release: prometheus-community
+  app: kube-prometheus-stack
+  release: prometheus-community
+  alert:
+    pinsPinErrorThreshold: 10
+    pinsPinQueuedThreshold: 10
+    pinsPinningThreshold: 10
 
 nodeSelector: {}
 tolerations: []
 affinity: {}
+