feat: delete statefulsets if volumeClaimTemplates changes

Author: cwrau

charts/tempo-distributed/templates/_helpers.tpl

@@ -298,3 +298,127 @@ Cluster name that shows up in dashboard metrics
 {{- define "tempo.clusterName" -}}
 {{ (include "tempo.calculatedConfig" . | fromYaml).cluster_name | default .Release.Name }}
 {{- end -}}
+
+{{- define "tempo.statefulset.recreateOnSizeChangeHook" -}}
+  {{- $renderedStatefulSets := list -}}
+  {{- range $renderedStatefulSet := include (print .context.Template.BasePath .pathToStatefulsetTemplate) .context | splitList "---" -}}
+    {{- with $renderedStatefulSet | fromYaml -}}
+      {{- $renderedStatefulSets = append $renderedStatefulSets . -}}
+    {{- end }}
+  {{- end -}}
+  {{- if $renderedStatefulSets }}
+    {{- range $newStatefulSet := $renderedStatefulSets -}}
+      {{- if $newStatefulSet.spec.volumeClaimTemplates }}
+        {{- $currentStatefulset := lookup $newStatefulSet.apiVersion $newStatefulSet.kind $newStatefulSet.metadata.namespace $newStatefulSet.metadata.name -}}
+        {{- $needsRecreation := false -}}
+        {{- if $currentStatefulset -}}
+          {{- if ne (len $newStatefulSet.spec.volumeClaimTemplates) (len $currentStatefulset.spec.volumeClaimTemplates) -}}
+            {{- $needsRecreation = true -}}
+          {{- else -}}
+            {{- range $index, $newVolumeClaimTemplate := $newStatefulSet.spec.volumeClaimTemplates -}}
+              {{- $currentVolumeClaimTemplateSpec := dict -}}
+                {{- range $oldVolumeClaimTemplate := $currentStatefulset.spec.volumeClaimTemplates -}}
+                  {{- if eq $oldVolumeClaimTemplate.metadata.name $newVolumeClaimTemplate.metadata.name -}}
+                    {{- $currentVolumeClaimTemplateSpec = $oldVolumeClaimTemplate.spec -}}
+                  {{- end -}}
+                {{- end }}
+              {{- if or (not $currentVolumeClaimTemplateSpec) (ne $newVolumeClaimTemplate.spec.resources.requests.storage $currentVolumeClaimTemplateSpec.resources.requests.storage) -}}
+                {{- $needsRecreation = true -}}
+              {{- end -}}
+            {{- end -}}
+          {{- end -}}
+        {{- end -}}
+        {{- if $needsRecreation }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ $newStatefulSet.metadata.name }}-recreate
+  namespace: {{ $newStatefulSet.metadata.namespace }}
+  labels:
+    {{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
+    app.kubernetes.io/component: statefulset-recreate-job
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  ttlSecondsAfterFinished: 300
+  template:
+    metadata:
+      name: {{ $newStatefulSet.metadata.name }}-recreate
+      labels:
+        {{- $newStatefulSet.metadata.labels | toYaml | nindent 8 }}
+    spec:
+      serviceAccountName: {{ $newStatefulSet.metadata.name }}-recreate
+      restartPolicy: OnFailure
+      containers:
+        - name: recreate
+          image: {{ printf "%s/rancher/kubectl:%s" (coalesce $.context.Values.global.image.registry "docker.io") $.context.Capabilities.KubeVersion.Version }}
+          command:
+            - kubectl
+            - delete
+            - statefulset
+            - {{ $newStatefulSet.metadata.name }}
+            - --cascade=orphan
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $newStatefulSet.metadata.name }}-recreate
+  namespace: {{ $newStatefulSet.metadata.namespace }}
+  labels:
+    {{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
+    app.kubernetes.io/component: statefulset-recreate-job
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $newStatefulSet.metadata.name }}-recreate
+  namespace: {{ $newStatefulSet.metadata.namespace }}
+  labels:
+    {{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
+    app.kubernetes.io/component: statefulset-recreate-job
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    resourceNames:
+      - {{ $newStatefulSet.metadata.name }}
+    verbs:
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $newStatefulSet.metadata.name }}-recreate
+  namespace: {{ $newStatefulSet.metadata.namespace }}
+  labels:
+    {{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
+    app.kubernetes.io/component: statefulset-recreate-job
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+subjects:
+  - kind: ServiceAccount
+    name: {{ $newStatefulSet.metadata.name }}-recreate
+    namespace: {{ $newStatefulSet.metadata.namespace }}
+roleRef:
+  kind: Role
+  name: {{ $newStatefulSet.metadata.name }}-recreate
+  apiGroup: rbac.authorization.k8s.io
+---
+        {{- end }}
+      {{ end }}
+    {{ end }}
+  {{ end }}
+{{- end -}}

charts/tempo-distributed/templates/ingester/statefulset-ingester-recreate-job.yaml

@@ -0,0 +1 @@
+{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/ingester/statefulset-ingester.yaml") -}}

charts/tempo-distributed/templates/ingester/statefulset-ingester.yaml

@@ -45,6 +45,9 @@ spec:
         {{- end }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
+        {{- if .Values.ingester.persistence.enabled }}
+        storage/size: {{ .Values.ingester.persistence.size | quote }}
+        {{- end }}
         {{- with .Values.tempo.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator-recreate-job.yaml

@@ -0,0 +1 @@
+{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/metrics-generator/statefulset-metrics-generator.yaml") -}}

charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator.yaml

@@ -36,6 +36,9 @@ spec:
         {{- end }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
+        {{- if .Values.metricsGenerator.persistence.enabled }}
+        storage/size: {{ .Values.metricsGenerator.persistence.size | quote }}
+        {{- end }}
         {{- with .Values.tempo.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}