added CRD upgrade job

AndrewChubatiuk · AndrewChubatiuk · commit 32e4b56a3326 · 2025-09-27T13:27:28.000+03:00
diff --git a/.gitignore b/.gitignore
@@ -9,8 +9,6 @@ test-charts/*/Chart.lock
 .cr-release-packages/**
 !charts/*/charts/crds
 !charts/*/charts/crds/**
-!charts/*/charts/crd-templates
-!charts/*/charts/crd-templates/**
 charts/**/charts/crds/_index.md
 .helm/
 .local/
diff --git a/charts/victoria-metrics-k8s-stack/values.yaml b/charts/victoria-metrics-k8s-stack/values.yaml
@@ -26,7 +26,8 @@ argocdReleaseOverride: ""
 victoria-metrics-operator:
   enabled: true
   crds:
-    plain: true
+    upgrade:
+      enabled: false
     cleanup:
       enabled: true
       image:
diff --git a/charts/victoria-metrics-operator-crds/Chart.yaml b/charts/victoria-metrics-operator-crds/Chart.yaml
@@ -34,5 +34,5 @@ annotations:
     - name: Changelog
       url: https://docs.victoriametrics.com/operator/changelog/
 dependencies:
-  - name: crd-templates
+  - name: crds
     version: "0.0.*"
diff --git a/charts/victoria-metrics-operator-crds/charts/crds/Chart.yaml b/charts/victoria-metrics-operator-crds/charts/crds/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v2
-name: crd-templates
+name: crds
 description: A subchart stores victoriametrics operator CRDs.
 type: application
 version: 0.0.0
diff --git a/charts/victoria-metrics-operator-crds/charts/crds/README.md b/charts/victoria-metrics-operator-crds/charts/crds/README.md
diff --git a/charts/victoria-metrics-operator-crds/charts/crds/crd.yaml b/charts/victoria-metrics-operator-crds/charts/crds/crd.yaml
diff --git a/charts/victoria-metrics-operator-crds/charts/crds/templates/crd.yaml b/charts/victoria-metrics-operator-crds/charts/crds/templates/crd.yaml
diff --git a/charts/victoria-metrics-operator-crds/charts/crds/values.yaml b/charts/victoria-metrics-operator-crds/charts/crds/values.yaml
diff --git a/charts/victoria-metrics-operator/CHANGELOG.md b/charts/victoria-metrics-operator/CHANGELOG.md
@@ -1,7 +1,10 @@
 ## Next release
 
+**Update note 1**: CRD template was replaced by CRD upgrade job. `Values.crds.plain` property was removed. Consider replacing it with `.Values.crds.upgrade.enabled: true` if automatic CRD upgrade is expected.
+**Update note 2**: Replaced `.Values.admissionWebhooks.enabledCRDValidation` map with `.Values.admissionWebhooks.disabledFor` list of CRD names to disable validation for.
+
 - Added `securityContext` to the `cleanup` job.
-- Removed `.Values.crds.plain` property, CRDs as templates are not supported by chart anymore.
+- Removed `.Values.crds.plain` property, CRDs as templates are not supported by chart anymore. Added upgrade job instead, which is enabled by `.Values.crds.upgrade.enabled` property.
 - Replaced `.Values.admissionWebhooks.enabledCRDValidation` with `.Values.admissionWebhooks.disabledFor` list of CRD names to disable validation for.
 
 ## 0.53.0
diff --git a/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 b/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2
diff --git a/charts/victoria-metrics-operator/charts/crds/templates/_helpers.tpl b/charts/victoria-metrics-operator/charts/crds/templates/_helpers.tpl
@@ -0,0 +1,13 @@
+{{- define "crds.upgrade.name" -}}
+{{- print (include "vm.plain.fullname" .) "-upgrade-crds" }}
+{{- end -}}
+
+{{- define "crds.upgrade.serviceAccountName" -}}
+{{- $Values := (.helm).Values | default .Values }}
+{{- $upgrade := $Values.upgrade }}
+{{- if $upgrade.serviceAccount.create -}}
+    {{ default (include "crds.upgrade.name" .) $upgrade.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" $upgrade.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/victoria-metrics-operator/charts/crds/templates/cm.yaml b/charts/victoria-metrics-operator/charts/crds/templates/cm.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.upgrade.enabled }}
+{{- $ctx := dict "helm" . }}
+{{- $upgrade := .Values.upgrade }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "crds.upgrade.serviceAccountName" $ctx }}
+  namespace: {{ template "vm.namespace" $ctx }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-2"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  {{- $_ := set $ctx "extraLabels" (dict "app.kubernetes.io/component" "upgrade-crds") }}
+  labels: {{ include "vm.labels" $ctx | nindent 4 }}
+  {{- $_ := unset $ctx "extraLabels" }}
+binaryData:
+  crd.yaml.bz2: {{ .Files.Get "files/crd.yaml.bz2" | b64enc }}
+{{- end }}
diff --git a/charts/victoria-metrics-operator/charts/crds/templates/job.yaml b/charts/victoria-metrics-operator/charts/crds/templates/job.yaml
@@ -0,0 +1,124 @@
+{{- if .Values.upgrade.enabled }}
+{{- $app := .Values.upgrade }}
+{{- if empty (($app.kubectl).image).tag }}
+  {{- $tag := regexSplit "[+-]" .Capabilities.KubeVersion.Version -1 | first -}}
+  {{- $_ := set $app.kubectl.image "tag" $tag }}
+{{- else if not (kindIs "string" (($app.kubectl).image).tag) }}
+  {{- fail "`crd.upgrade.kubectl.image.tag` is not string, most probably you need to enquote provided value" -}}
+{{- end }}
+{{- $ctx := dict "helm" . "noEnterprise" true }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "crds.upgrade.name" $ctx }}
+  namespace: {{ template "vm.namespace" $ctx }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "5"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    {{- with $app.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- $_ := set $ctx "extraLabels" $app.labels }}
+  {{- $_ := set $ctx.extraLabels "app.kubernetes.io/component" "upgrade-crds" }}
+  labels: {{ include "vm.labels" $ctx | nindent 4 }}
+  {{- $_ := unset $ctx "extraLabels" }}
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      {{- with $app.podLabels }}
+      labels: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $app.podAnnotations }}
+      annotations: {{ toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- with (.Values.imagePullSecrets | default .Values.global.imagePullSecrets) }}
+      imagePullSecrets: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "crds.upgrade.serviceAccountName" . }}
+      {{- if $app.podSecurityContext.enabled }}
+      securityContext: {{ include "vm.securityContext" (dict "securityContext" $app.podSecurityContext "helm" .) | nindent 8 }}
+      {{- end }}
+      initContainers:
+        - name: busybox
+          {{- $_ := set $ctx "appKey" (list "upgrade" "busybox") }}
+          image: {{ include "vm.image" $ctx }}
+          imagePullPolicy: {{ $app.busybox.image.pullPolicy }}
+          workingDir: /tmp/
+          command:
+            - sh
+          args:
+            - -c
+            - bzcat /crds/crd.yaml.bz2 > /tmp/crd.yaml
+          {{- with $app.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with $app.securityContext }}
+          securityContext: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - mountPath: /crds/
+              name: crds
+            - mountPath: /tmp/
+              name: tmp
+            {{- with $app.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with $app.env }}
+          env: {{ toYaml . | nindent 12 }}
+          {{- end }}
+      containers:
+        - name: kubectl
+          {{- $_ := set $ctx "appKey" (list "upgrade" "kubectl") }}
+          image: {{ include "vm.image" $ctx }}
+          imagePullPolicy: {{ $app.kubectl.image.pullPolicy }}
+          command:
+            - kubectl
+          args:
+            - apply
+            - --server-side
+            {{- if $app.forceConflicts }}
+            - --force-conflicts
+            {{- end }}
+            - --filename
+            - /tmp/crd.yaml
+          {{- with $app.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with $app.securityContext }}
+          securityContext: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - mountPath: /tmp/
+              name: tmp
+            {{- with $app.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- with $app.env }}
+          env: {{ toYaml . | nindent 12 }}
+          {{- end }}
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: crds
+          configMap:
+            name: {{ template "crds.upgrade.name" . }}
+        {{- with $app.extraVolumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      restartPolicy: OnFailure
+      {{- with $app.nodeSelector }}
+      nodeSelector: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $app.tolerations }}
+      tolerations: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $app.affinity }}
+      affinity: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $app.topologySpreadConstraints }}
+      topologySpreadConstraints: {{ toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}
diff --git a/charts/victoria-metrics-operator/charts/crds/templates/role.yaml b/charts/victoria-metrics-operator/charts/crds/templates/role.yaml
@@ -0,0 +1,52 @@
+{{- if .Values.upgrade.enabled }}
+{{- $ctx := dict "helm" . }}
+{{- $_ := set $ctx "extraLabels" (dict "app.kubernetes.io/component" "upgrade-crds") }}
+{{- $labels := include "vm.labels" $ctx }}
+{{- $_ := unset $ctx "extraLabels" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "crds.upgrade.name" . }}
+  namespace: {{ template "vm.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels: {{ $labels | nindent 4 }}
+  {{- $crds := .Files.Get "crds/crd.yaml" | splitList "---" }}
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - "customresourcedefinitions"
+    verbs:
+      - create
+      - patch
+      - update
+      - get
+      - list
+    resourceNames:
+      {{- range $crds }}
+        {{- $crd := fromYaml . }}
+      - {{ $crd.metadata.name }}
+      {{- end }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "crds.upgrade.name" . }}
+  namespace: {{ template "vm.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels: {{ $labels | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    namespace: {{ template "vm.namespace" . }}
+    name: {{ template "crds.upgrade.serviceAccountName" . }}
+roleRef:
+  kind: ClusterRole
+  name: {{ template "crds.upgrade.name" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/charts/victoria-metrics-operator/charts/crds/templates/serviceaccount.yaml b/charts/victoria-metrics-operator/charts/crds/templates/serviceaccount.yaml
@@ -0,0 +1,24 @@
+{{- $upgrade := .Values.upgrade }}
+{{- if and $upgrade.enabled $upgrade.serviceAccount.create }}
+{{- $ctx := dict "helm" . }}
+{{- $fullname := include "vm.plain.fullname" $ctx }}
+{{- $ns := include "vm.namespace" $ctx }}
+{{- $sa := $upgrade.serviceAccount }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ $sa.automountServiceAccountToken }}
+metadata:
+  name: {{ include "crds.upgrade.name" . }}
+  namespace: {{ $ns }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback
+    "helm.sh/hook-weight": "-4"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    {{- with $sa.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- $_ := set $ctx "extraLabels" $sa.labels }}
+  {{- $_ := set $ctx.extraLabels "app.kubernetes.io/component" "upgrade-crds" }}
+  labels: {{ include "vm.labels" $ctx | nindent 4 }}
+  {{- $_ := unset $ctx "extraLabels" }}
+{{- end }}
diff --git a/charts/victoria-metrics-operator/charts/crds/values.yaml b/charts/victoria-metrics-operator/charts/crds/values.yaml
@@ -0,0 +1,2 @@
+upgrade:
+  enabled: true
diff --git a/charts/victoria-metrics-operator/templates/_crd.tpl b/charts/victoria-metrics-operator/templates/_crd.tpl
diff --git a/charts/victoria-metrics-operator/templates/cleanup.yaml b/charts/victoria-metrics-operator/templates/cleanup.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.crds.enabled .Values.crds.cleanup.enabled }}
+{{- if .Values.crds.cleanup.enabled }}
 {{- $app := .Values.crds.cleanup }}
 {{- if empty ($app.image).tag }}
   {{- $tag := regexSplit "[+-]" .Capabilities.KubeVersion.Version -1 | first -}}
@@ -36,9 +36,14 @@ spec:
           {{- if .Values.securityContext.enabled }}
           securityContext: {{ include "vm.securityContext" (dict "securityContext" .Values.securityContext "helm" .) | nindent 12 }}
           {{- end }}
+          {{- $names := default list }}
+          {{- $crds := fromYaml (include "vm-operator.crds" .) }}
+          {{- range $crd := $crds.crds }}
+            {{- $names = append $names $crd.names.singular }}
+          {{- end }}
           args:
             - delete
-            - {{ (keys .Values.admissionWebhooks.enabledCRDValidation) | sortAlpha | join "," }}
+            - {{ $names | sortAlpha | join "," }}
             - --all
             - --ignore-not-found=true
       restartPolicy: OnFailure
diff --git a/charts/victoria-metrics-operator/templates/crb.yaml b/charts/victoria-metrics-operator/templates/crb.yaml
@@ -21,7 +21,7 @@ roleRef:
   name: {{ $fullname }}
   apiGroup: rbac.authorization.k8s.io
 {{- end -}}
-{{- if and .Values.crds.enabled .Values.crds.cleanup.enabled }}
+{{- if .Values.crds.cleanup.enabled }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/charts/victoria-metrics-operator/templates/role.yaml b/charts/victoria-metrics-operator/templates/role.yaml
@@ -177,7 +177,7 @@ rules:
 {{ toYaml . }}
 {{- end }}
 {{- end }}
-{{- if and .Values.crds.enabled .Values.crds.cleanup.enabled }}
+{{- if .Values.crds.cleanup.enabled }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/charts/victoria-metrics-operator/templates/serviceaccount.yaml b/charts/victoria-metrics-operator/templates/serviceaccount.yaml
@@ -15,7 +15,7 @@ metadata:
   {{- end }}
 automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
 {{- end }}
-{{- if and .Values.crds.enabled .Values.crds.cleanup.enabled }}
+{{- if .Values.crds.cleanup.enabled }}
 ---
 apiVersion: v1
 kind: ServiceAccount
diff --git a/charts/victoria-metrics-operator/templates/webhook.yaml b/charts/victoria-metrics-operator/templates/webhook.yaml
@@ -33,7 +33,7 @@ webhooks:
       caBundle: {{ $tls.caCert }}
       {{- end }}
     failurePolicy: {{ $.Values.admissionWebhooks.policy }}
-    name: '{{ $crd.names.singular }}.{{ $crd.group }}'
+    name: '{{ $crd.names.plural }}.{{ $crd.group }}'
     admissionReviewVersions:
       - {{ $version }}
     sideEffects: None
diff --git a/charts/victoria-metrics-operator/values.yaml b/charts/victoria-metrics-operator/values.yaml
