Skip to content

Commit fac6559

Browse files
cwraucwrau
committed
feat: delete statefulsets if volumeClaimTemplates changes
1 parent 24bcd88 commit fac6559

File tree

7 files changed

+144
-2
lines changed

7 files changed

+144
-2
lines changed

‎charts/tempo-distributed/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -596,8 +596,9 @@ The memcached default args are removed and should be provided manually. The sett
596596
| ingester.image.tag | string | `nil` | Docker image tag for the ingester image. Overrides `tempo.image.tag` |
597597
| ingester.initContainers | list | `[]` | |
598598
| ingester.nodeSelector | object | `{}` | Node selector for ingester pods |
599-
| ingester.persistence | object | `{"annotations":{},"enabled":false,"inMemory":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for ingester |
599+
| ingester.persistence | object | `{"annotations":{},"enableStatefulSetRecreationForSizeChange":false,"enabled":false,"inMemory":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for ingester |
600600
| ingester.persistence.annotations | object | `{}` | Annotations for ingester's persist volume claim |
601+
| ingester.persistence.enableStatefulSetRecreationForSizeChange | bool | `false` | Enable StatefulSetRecreation for changes to PVC size |
601602
| ingester.persistence.enabled | bool | `false` | Enable creating PVCs which is required when using boltdb-shipper |
602603
| ingester.persistence.inMemory | bool | `false` | use emptyDir with ramdisk instead of PVC. **Please note that all data in ingester will be lost on pod restart** |
603604
| ingester.persistence.labels | object | `{}` | Labels for ingester's persist volume claim |
@@ -745,8 +746,9 @@ The memcached default args are removed and should be provided manually. The sett
745746
| metricsGenerator.maxUnavailable | int | `1` | Pod Disruption Budget maxUnavailable |
746747
| metricsGenerator.minReadySeconds | int | `10` | Minimum number of seconds for which a newly created Pod should be ready without any of its containers crashing/terminating |
747748
| metricsGenerator.nodeSelector | object | `{}` | Node selector for metrics-generator pods |
748-
| metricsGenerator.persistence | object | `{"annotations":{},"enabled":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for metrics-generator |
749+
| metricsGenerator.persistence | object | `{"annotations":{},"enableStatefulSetRecreationForSizeChange":false,"enabled":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for metrics-generator |
749750
| metricsGenerator.persistence.annotations | object | `{}` | Annotations for metrics generator PVCs |
751+
| metricsGenerator.persistence.enableStatefulSetRecreationForSizeChange | bool | `false` | Enable StatefulSetRecreation for changes to PVC size |
750752
| metricsGenerator.persistence.enabled | bool | `false` | Enable creating PVCs if you have kind set to StatefulSet. This disables using local disk or memory configured in walEmptyDir |
751753
| metricsGenerator.persistence.labels | object | `{}` | Labels for metrics generator PVCs |
752754
| metricsGenerator.persistence.storageClass | string | `nil` | Storage class to be used. If defined, storageClassName: <storageClass>. If set to "-", storageClassName: "", which disables dynamic provisioning. If empty or set to null, no storageClassName spec is set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). |

‎charts/tempo-distributed/templates/_helpers.tpl‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,3 +298,127 @@ Cluster name that shows up in dashboard metrics
298298
{{- define "tempo.clusterName" -}}
299299
{{ (include "tempo.calculatedConfig" . | fromYaml).cluster_name | default .Release.Name }}
300300
{{- end -}}
301+
302+
{{- define "tempo.statefulset.recreateOnSizeChangeHook" -}}
303+
{{- $renderedStatefulSets := list -}}
304+
{{- range $renderedStatefulSet := include (print .context.Template.BasePath .pathToStatefulsetTemplate) .context | splitList "---" -}}
305+
{{- with $renderedStatefulSet | fromYaml -}}
306+
{{- $renderedStatefulSets = append $renderedStatefulSets . -}}
307+
{{- end }}
308+
{{- end -}}
309+
{{- if $renderedStatefulSets }}
310+
{{- range $newStatefulSet := $renderedStatefulSets -}}
311+
{{- if $newStatefulSet.spec.volumeClaimTemplates }}
312+
{{- $currentStatefulset := lookup $newStatefulSet.apiVersion $newStatefulSet.kind $newStatefulSet.metadata.namespace $newStatefulSet.metadata.name -}}
313+
{{- $needsRecreation := false -}}
314+
{{- if $currentStatefulset -}}
315+
{{- if ne (len $newStatefulSet.spec.volumeClaimTemplates) (len $currentStatefulset.spec.volumeClaimTemplates) -}}
316+
{{- $needsRecreation = true -}}
317+
{{- else -}}
318+
{{- range $index, $newVolumeClaimTemplate := $newStatefulSet.spec.volumeClaimTemplates -}}
319+
{{- $currentVolumeClaimTemplateSpec := dict -}}
320+
{{- range $oldVolumeClaimTemplate := $currentStatefulset.spec.volumeClaimTemplates -}}
321+
{{- if eq $oldVolumeClaimTemplate.metadata.name $newVolumeClaimTemplate.metadata.name -}}
322+
{{- $currentVolumeClaimTemplateSpec = $oldVolumeClaimTemplate.spec -}}
323+
{{- end -}}
324+
{{- end }}
325+
{{- if or (not $currentVolumeClaimTemplateSpec) (ne $newVolumeClaimTemplate.spec.resources.requests.storage $currentVolumeClaimTemplateSpec.resources.requests.storage) -}}
326+
{{- $needsRecreation = true -}}
327+
{{- end -}}
328+
{{- end -}}
329+
{{- end -}}
330+
{{- end -}}
331+
{{- if $needsRecreation }}
332+
apiVersion: batch/v1
333+
kind: Job
334+
metadata:
335+
name: {{ $newStatefulSet.metadata.name }}-recreate
336+
namespace: {{ $newStatefulSet.metadata.namespace }}
337+
labels:
338+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
339+
app.kubernetes.io/component: statefulset-recreate-job
340+
annotations:
341+
"helm.sh/hook": pre-upgrade
342+
"helm.sh/hook-weight": "-5"
343+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
344+
spec:
345+
ttlSecondsAfterFinished: 300
346+
template:
347+
metadata:
348+
name: {{ $newStatefulSet.metadata.name }}-recreate
349+
labels:
350+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 8 }}
351+
spec:
352+
serviceAccountName: {{ $newStatefulSet.metadata.name }}-recreate
353+
restartPolicy: OnFailure
354+
containers:
355+
- name: recreate
356+
image: {{ printf "%s/rancher/kubectl:%s" (coalesce $.context.Values.global.image.registry "docker.io") $.context.Capabilities.KubeVersion.Version }}
357+
command:
358+
- kubectl
359+
- delete
360+
- statefulset
361+
- {{ $newStatefulSet.metadata.name }}
362+
- --cascade=orphan
363+
---
364+
apiVersion: v1
365+
kind: ServiceAccount
366+
metadata:
367+
name: {{ $newStatefulSet.metadata.name }}-recreate
368+
namespace: {{ $newStatefulSet.metadata.namespace }}
369+
labels:
370+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
371+
app.kubernetes.io/component: statefulset-recreate-job
372+
annotations:
373+
"helm.sh/hook": pre-upgrade
374+
"helm.sh/hook-weight": "-10"
375+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
376+
---
377+
apiVersion: rbac.authorization.k8s.io/v1
378+
kind: Role
379+
metadata:
380+
name: {{ $newStatefulSet.metadata.name }}-recreate
381+
namespace: {{ $newStatefulSet.metadata.namespace }}
382+
labels:
383+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
384+
app.kubernetes.io/component: statefulset-recreate-job
385+
annotations:
386+
"helm.sh/hook": pre-upgrade
387+
"helm.sh/hook-weight": "-10"
388+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
389+
rules:
390+
- apiGroups:
391+
- apps
392+
resources:
393+
- statefulsets
394+
resourceNames:
395+
- {{ $newStatefulSet.metadata.name }}
396+
verbs:
397+
- delete
398+
---
399+
apiVersion: rbac.authorization.k8s.io/v1
400+
kind: RoleBinding
401+
metadata:
402+
name: {{ $newStatefulSet.metadata.name }}-recreate
403+
namespace: {{ $newStatefulSet.metadata.namespace }}
404+
labels:
405+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
406+
app.kubernetes.io/component: statefulset-recreate-job
407+
annotations:
408+
"helm.sh/hook": pre-upgrade
409+
"helm.sh/hook-weight": "-10"
410+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
411+
subjects:
412+
- kind: ServiceAccount
413+
name: {{ $newStatefulSet.metadata.name }}-recreate
414+
namespace: {{ $newStatefulSet.metadata.namespace }}
415+
roleRef:
416+
kind: Role
417+
name: {{ $newStatefulSet.metadata.name }}-recreate
418+
apiGroup: rbac.authorization.k8s.io
419+
---
420+
{{- end }}
421+
{{ end }}
422+
{{ end }}
423+
{{ end }}
424+
{{- end -}}

‎charts/tempo-distributed/templates/ingester/statefulset-ingester-recreate-job.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{{- if and .Values.ingester.persistence.enabled .Values.ingester.persistence.enableStatefulSetRecreationForSizeChange -}}
2+
{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/ingester/statefulset-ingester.yaml") -}}
3+
{{- end -}}

‎charts/tempo-distributed/templates/ingester/statefulset-ingester.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ spec:
4545
{{- end }}
4646
annotations:
4747
checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
48+
{{- if .Values.ingester.persistence.enabled }}
49+
storage/size: {{ .Values.ingester.persistence.size | quote }}
50+
{{- end }}
4851
{{- with .Values.tempo.podAnnotations }}
4952
{{- toYaml . | nindent 8 }}
5053
{{- end }}

‎charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator-recreate-job.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{{- if and .Values.metricsGenerator.persistence.enabled .Values.metricsGenerator.persistence.enableStatefulSetRecreationForSizeChange -}}
2+
{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/metrics-generator/statefulset-metrics-generator.yaml") -}}
3+
{{- end -}}

‎charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ spec:
3636
{{- end }}
3737
annotations:
3838
checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
39+
{{- if .Values.metricsGenerator.persistence.enabled }}
40+
storage/size: {{ .Values.metricsGenerator.persistence.size | quote }}
41+
{{- end }}
3942
{{- with .Values.tempo.podAnnotations }}
4043
{{- toYaml . | nindent 8 }}
4144
{{- end }}

‎charts/tempo-distributed/values.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,8 @@ ingester:
217217
persistence:
218218
# -- Enable creating PVCs which is required when using boltdb-shipper
219219
enabled: false
220+
# -- Enable StatefulSetRecreation for changes to PVC size
221+
enableStatefulSetRecreationForSizeChange: false
220222
# -- use emptyDir with ramdisk instead of PVC. **Please note that all data in ingester will be lost on pod restart**
221223
inMemory: false
222224
# -- Size of persistent or memory disk
@@ -401,6 +403,8 @@ metricsGenerator:
401403
persistence:
402404
# -- Enable creating PVCs if you have kind set to StatefulSet. This disables using local disk or memory configured in walEmptyDir
403405
enabled: false
406+
# -- Enable StatefulSetRecreation for changes to PVC size
407+
enableStatefulSetRecreationForSizeChange: false
404408
size: 10Gi
405409
# -- Storage class to be used.
406410
# If defined, storageClassName: <storageClass>.

0 commit comments

Comments
 (0)