From 1426851c04cd0be8656e0947e795a90692aed7b0 Mon Sep 17 00:00:00 2001 From: Eugene Bosiakov Date: Sat, 16 Nov 2024 14:41:16 +0200 Subject: [PATCH 1/2] Implement nodeSelector + taint support --- api/v1alpha1/sleepcycle_types.go | 105 ++++++++++++++++++++- controllers/sleepcycle_runners_cronjobs.go | 23 +++-- controllers/sleepcycle_utils.go | 26 ++++- controllers/sleepcycles_rbac.go | 1 + 4 files changed, 147 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/sleepcycle_types.go b/api/v1alpha1/sleepcycle_types.go index 0418a76..0b845af 100644 --- a/api/v1alpha1/sleepcycle_types.go +++ b/api/v1alpha1/sleepcycle_types.go @@ -17,6 +17,9 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,9 +66,109 @@ type SleepCycleSpec struct { // +kubebuilder:validation:ExclusiveMaximum=false FailedJobsHistoryLimit int32 `json:"failedJobsHistoryLimit,omitempty"` + Runner RunnerConfig `json:"runner,omitempty"` +} + +type Metadata struct { + // Additionnal annotation to merge to the resource associated + // https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + Annotations map[string]string `json:"annotations,omitempty"` + // Additionnal labels to merge to the resource associated + // https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + Labels map[string]string `json:"labels,omitempty"` +} + +// RunnerConfig defines the configuration of runner image +type RunnerConfig struct { // +kubebuilder:validation:Optional // +kubebuilder:default:="akyriako78/rekuberate-io-sleepcycles-runners" - RunnerImage string `json:"runnerImage,omitempty"` + Image string `json:"runnerImage,omitempty"` + + // RunAsUser define the id of the user to run in the image + // +kubebuilder:validation:Minimum=1 + RunAsUser *int64 `json:"runAsUser,omitempty"` + + // imagePullPolicy define the pull policy for docker image + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + + // resourceRequirements works exactly like Container resources, the user can specify the limit and the requests + // through this property + // https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + // +kubebuilder:validation:Optional + ResourcesRequirements *corev1.ResourceRequirements `json:"resourcesRequirements,omitempty"` + // imagePullSecrets specifies the secret to use when using private registry + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#localobjectreference-v1-core + // +kubebuilder:validation:Optional + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + // nodeSelector can be specified, which set the pod to fit on a node + // https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + // +kubebuilder:validation:Optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // tolerations can be specified, which set the pod's tolerations + // https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts + // +kubebuilder:validation:Optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // podMetadata allows to add additionnal metadata to the node pods + PodMetadata Metadata `json:"podMetadata,omitempty"` + + // priorityClassName can be used to set the priority class applied to the node + // +optional + PriorityClassName *string `json:"priorityClassName,omitempty"` +} + +// GetResources returns the sleepcycle runner Kubernetes resource. +func (r *RunnerConfig) GetResources() *v1.ResourceRequirements { + if r.ResourcesRequirements != nil { + return r.ResourcesRequirements + } + return &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("10m"), + "memory": resource.MustParse("128mb"), + }, + Requests: v1.ResourceList{ + "cpu": resource.MustParse("10m"), + "memory": resource.MustParse("128mb"), + }, + } +} + +// GetTolerations returns the tolerations for the given node. +func (r *RunnerConfig) GetTolerations() []corev1.Toleration { + return r.Tolerations +} + +// GetNodeSelector returns the node selector for the given node. +func (r *RunnerConfig) GetNodeSelector() map[string]string { + return r.NodeSelector +} + +// GetImagePullSecrets returns the list of Secrets needed to pull Containers images from private repositories. +func (r *RunnerConfig) GetImagePullSecrets() []corev1.LocalObjectReference { + return r.ImagePullSecrets +} + +// GetImagePullPolicy returns the image pull policy to pull containers images. +func (r *RunnerConfig) GetImagePullPolicy() corev1.PullPolicy { + return r.ImagePullPolicy +} + +func (r *RunnerConfig) GetPodAnnotations() map[string]string { + return r.PodMetadata.Annotations +} + +// GetNodeLabels returns additional labels configured to be applied to each nifi node. +func (r *RunnerConfig) GetPodLabels() map[string]string { + return r.PodMetadata.Labels +} + +// GetPriorityClass returns the name of the priority class to use for the given node. +func (r *RunnerConfig) GetPriorityClass() string { + if r.PriorityClassName != nil { + return *r.PriorityClassName + } + return "" } // SleepCycleStatus defines the observed state of SleepCycle diff --git a/controllers/sleepcycle_runners_cronjobs.go b/controllers/sleepcycle_runners_cronjobs.go index 1387626..1422066 100644 --- a/controllers/sleepcycle_runners_cronjobs.go +++ b/controllers/sleepcycle_runners_cronjobs.go @@ -3,6 +3,9 @@ package controllers import ( "context" "fmt" + "strconv" + "strings" + "github.com/go-logr/logr" corev1alpha1 "github.com/rekuberate-io/sleepcycles/api/v1alpha1" batchv1 "k8s.io/api/batch/v1" @@ -10,8 +13,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "strconv" - "strings" ) var ( @@ -74,8 +75,8 @@ func (r *SleepCycleReconciler) createCronJob( } labels := make(map[string]string) - labels[OwnedBy] = fmt.Sprintf("%s", sleepcycle.Name) - labels[Target] = fmt.Sprintf("%s", targetMeta.Name) + labels[OwnedBy] = sleepcycle.Name + labels[Target] = targetMeta.Name labels[TargetKind] = targetKind annotations := make(map[string]string) @@ -110,8 +111,10 @@ func (r *SleepCycleReconciler) createCronJob( Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: cronObjectKey.Name, - Image: sleepcycle.Spec.RunnerImage, + Name: cronObjectKey.Name, + Image: sleepcycle.Spec.Runner.Image, + ImagePullPolicy: sleepcycle.Spec.Runner.GetImagePullPolicy(), + Resources: *sleepcycle.Spec.Runner.GetResources(), Env: []v1.EnvVar{ { Name: "MY_POD_NAME", @@ -136,6 +139,14 @@ func (r *SleepCycleReconciler) createCronJob( }, RestartPolicy: v1.RestartPolicyOnFailure, ServiceAccountName: serviceAccountName, + ImagePullSecrets: sleepcycle.Spec.Runner.GetImagePullSecrets(), + Tolerations: sleepcycle.Spec.Runner.GetTolerations(), + PriorityClassName: sleepcycle.Spec.Runner.GetPriorityClass(), + NodeSelector: sleepcycle.Spec.Runner.GetNodeSelector(), + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: MergeLabels(labels, sleepcycle.Spec.Runner.GetPodLabels()), + Annotations: MergeAnnotations(annotations, sleepcycle.Spec.Runner.GetPodAnnotations()), }, }, BackoffLimit: &backOffLimit, diff --git a/controllers/sleepcycle_utils.go b/controllers/sleepcycle_utils.go index 3597ca3..64e01f3 100644 --- a/controllers/sleepcycle_utils.go +++ b/controllers/sleepcycle_utils.go @@ -3,10 +3,11 @@ package controllers import ( "crypto/rand" "encoding/base64" + "strings" + corev1alpha1 "github.com/rekuberate-io/sleepcycles/api/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" ) const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" @@ -57,3 +58,26 @@ func (r *SleepCycleReconciler) recordEvent(sleepCycle *corev1alpha1.SleepCycle, r.Recorder.Event(sleepCycle, eventType, reason, strings.ToLower(message)) } + +// MergeLabels merges two given labels. +func MergeLabels(l ...map[string]string) map[string]string { + res := make(map[string]string) + + for _, v := range l { + for lKey, lValue := range v { + res[lKey] = lValue + } + } + return res +} + +func MergeAnnotations(annotations ...map[string]string) map[string]string { + rtn := make(map[string]string) + for _, a := range annotations { + for k, v := range a { + rtn[k] = v + } + } + + return rtn +} diff --git a/controllers/sleepcycles_rbac.go b/controllers/sleepcycles_rbac.go index 848f973..edc47df 100644 --- a/controllers/sleepcycles_rbac.go +++ b/controllers/sleepcycles_rbac.go @@ -3,6 +3,7 @@ package controllers import ( "context" "fmt" + "github.com/pkg/errors" corev1alpha1 "github.com/rekuberate-io/sleepcycles/api/v1alpha1" v1 "k8s.io/api/core/v1" From f3b197b82e103f4a1c090b25fdc6f92dc3cac42b Mon Sep 17 00:00:00 2001 From: Eugene Bosiakov Date: Sat, 16 Nov 2024 14:55:24 +0200 Subject: [PATCH 2/2] add to spec --- api/v1alpha1/sleepcycle_types.go | 31 +++++-- .../sleepcycles/templates/sleepcycle-crd.yaml | 85 ++++++++++++++++++- .../bases/core.rekuberate.io_sleepcycles.yaml | 85 ++++++++++++++++++- controllers/sleepcycle_runners_cronjobs.go | 4 + 4 files changed, 190 insertions(+), 15 deletions(-) diff --git a/api/v1alpha1/sleepcycle_types.go b/api/v1alpha1/sleepcycle_types.go index 0b845af..d6daf9e 100644 --- a/api/v1alpha1/sleepcycle_types.go +++ b/api/v1alpha1/sleepcycle_types.go @@ -84,26 +84,30 @@ type RunnerConfig struct { // +kubebuilder:default:="akyriako78/rekuberate-io-sleepcycles-runners" Image string `json:"runnerImage,omitempty"` - // RunAsUser define the id of the user to run in the image - // +kubebuilder:validation:Minimum=1 - RunAsUser *int64 `json:"runAsUser,omitempty"` - // imagePullPolicy define the pull policy for docker image ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - // resourceRequirements works exactly like Container resources, the user can specify the limit and the requests - // through this property - // https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ - // +kubebuilder:validation:Optional - ResourcesRequirements *corev1.ResourceRequirements `json:"resourcesRequirements,omitempty"` // imagePullSecrets specifies the secret to use when using private registry // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#localobjectreference-v1-core // +kubebuilder:validation:Optional ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + + // RunAsUser define the id of the user to run in the image + // +kubebuilder:validation:Minimum=1 + RunAsUser *int64 `json:"runAsUser,omitempty"` + // nodeSelector can be specified, which set the pod to fit on a node // https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector // +kubebuilder:validation:Optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // resourceRequirements works exactly like Container resources, the user can specify the limit and the requests + // through this property + // https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + // +kubebuilder:validation:Optional + ResourcesRequirements *corev1.ResourceRequirements `json:"resourcesRequirements,omitempty"` + // tolerations can be specified, which set the pod's tolerations // https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#concepts // +kubebuilder:validation:Optional @@ -134,6 +138,15 @@ func (r *RunnerConfig) GetResources() *v1.ResourceRequirements { } } +func (r *RunnerConfig) GetRunAsUser() *int64 { + var defaultUserID int64 = 1000 + if r.RunAsUser != nil { + return r.RunAsUser + } + + return func(i int64) *int64 { return &i }(defaultUserID) +} + // GetTolerations returns the tolerations for the given node. func (r *RunnerConfig) GetTolerations() []corev1.Toleration { return r.Tolerations diff --git a/charts/sleepcycles/templates/sleepcycle-crd.yaml b/charts/sleepcycles/templates/sleepcycle-crd.yaml index a5a6664..112b7f9 100644 --- a/charts/sleepcycles/templates/sleepcycle-crd.yaml +++ b/charts/sleepcycles/templates/sleepcycle-crd.yaml @@ -65,9 +65,88 @@ spec: maximum: 3 minimum: 1 type: integer - runnerImage: - default: akyriako78/rekuberate-io-sleepcycles-runners - type: string + runner: + image: + default: akyriako78/rekuberate-io-sleepcycles-runners + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + runAsUser: + format: int64 + minimum: 1 + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + podMetadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + priorityClassName: + type: string + resourcesRequirements: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array shutdown: pattern: (^((\*\/)?([0-5]?[0-9])((\,|\-|\/)([0-5]?[0-9]))*|\*)\s+((\*\/)?((2[0-3]|1[0-9]|[0-9]|00))((\,|\-|\/)(2[0-3]|1[0-9]|[0-9]|00))*|\*)\s+((\*\/)?([1-9]|[12][0-9]|3[01])((\,|\-|\/)([1-9]|[12][0-9]|3[01]))*|\*)\s+((\*\/)?([1-9]|1[0-2])((\,|\-|\/)([1-9]|1[0-2]))*|\*|(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|des))\s+((\*\/)?[0-6]((\,|\-|\/)[0-6])*|\*|00|(sun|mon|tue|wed|thu|fri|sat))\s*$)|@(annually|yearly|monthly|weekly|daily|hourly|reboot) type: string diff --git a/config/crd/bases/core.rekuberate.io_sleepcycles.yaml b/config/crd/bases/core.rekuberate.io_sleepcycles.yaml index 489c7ab..69ce538 100644 --- a/config/crd/bases/core.rekuberate.io_sleepcycles.yaml +++ b/config/crd/bases/core.rekuberate.io_sleepcycles.yaml @@ -65,9 +65,88 @@ spec: maximum: 3 minimum: 1 type: integer - runnerImage: - default: akyriako78/rekuberate-io-sleepcycles-runners - type: string + runner: + image: + default: akyriako78/rekuberate-io-sleepcycles-runners + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + runAsUser: + format: int64 + minimum: 1 + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + podMetadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + priorityClassName: + type: string + resourcesRequirements: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array shutdown: pattern: (^((\*\/)?([0-5]?[0-9])((\,|\-|\/)([0-5]?[0-9]))*|\*)\s+((\*\/)?((2[0-3]|1[0-9]|[0-9]|00))((\,|\-|\/)(2[0-3]|1[0-9]|[0-9]|00))*|\*)\s+((\*\/)?([1-9]|[12][0-9]|3[01])((\,|\-|\/)([1-9]|[12][0-9]|3[01]))*|\*)\s+((\*\/)?([1-9]|1[0-2])((\,|\-|\/)([1-9]|1[0-2]))*|\*|(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|des))\s+((\*\/)?[0-6]((\,|\-|\/)[0-6])*|\*|00|(sun|mon|tue|wed|thu|fri|sat))\s*$)|@(annually|yearly|monthly|weekly|daily|hourly|reboot) type: string diff --git a/controllers/sleepcycle_runners_cronjobs.go b/controllers/sleepcycle_runners_cronjobs.go index 1422066..51779af 100644 --- a/controllers/sleepcycle_runners_cronjobs.go +++ b/controllers/sleepcycle_runners_cronjobs.go @@ -143,6 +143,10 @@ func (r *SleepCycleReconciler) createCronJob( Tolerations: sleepcycle.Spec.Runner.GetTolerations(), PriorityClassName: sleepcycle.Spec.Runner.GetPriorityClass(), NodeSelector: sleepcycle.Spec.Runner.GetNodeSelector(), + SecurityContext: &v1.PodSecurityContext{ + RunAsUser: sleepcycle.Spec.Runner.GetRunAsUser(), + RunAsNonRoot: func(b bool) *bool { return &b }(true), + }, }, ObjectMeta: metav1.ObjectMeta{ Labels: MergeLabels(labels, sleepcycle.Spec.Runner.GetPodLabels()),