From 4b9fe2fb1b2f3084687498bf7c56f4aeb1dd3330 Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Thu, 5 Apr 2018 13:56:45 +0200 Subject: [PATCH 1/3] Review snapshots cronjob that was using empty namespaces, promote cronjobs to v1beta1 --- pkg/processor/processor.go | 16 +++++------ pkg/snapshot/scheduler.go | 54 ++++++++++++++++++++++++-------------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 81183ea15..75bbc3921 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -209,14 +209,14 @@ func (p *Processor) refreshClusters() error { }, }, Scheduler: snapshot.New( - cluster.Spec.Scheduler.S3bucketName, - cluster.Spec.Scheduler.CronSchedule, - cluster.Spec.Scheduler.Enabled, - cluster.Spec.Scheduler.Auth.UserName, - cluster.Spec.Scheduler.Auth.Password, - cluster.Spec.Scheduler.ElasticURL, - cluster.Spec.Scheduler.ClusterName, - cluster.Spec.Scheduler.Namespace, + cluster.Spec.Snapshot.BucketName, + cluster.Spec.Snapshot.CronSchedule, + cluster.Spec.Snapshot.SchedulerEnabled, + cluster.Spec.Snapshot.Authentication.UserName, + cluster.Spec.Snapshot.Authentication.Password, + p.k8sclient.GetClientServiceNameFullDNS(cluster.ObjectMeta.Name, cluster.ObjectMeta.Namespace), + cluster.ObjectMeta.Name, + cluster.ObjectMeta.Namespace, p.k8sclient.Kclient, ), } diff --git a/pkg/snapshot/scheduler.go b/pkg/snapshot/scheduler.go index 9b191ca43..bb6f01673 100644 --- a/pkg/snapshot/scheduler.go +++ b/pkg/snapshot/scheduler.go @@ -26,6 +26,7 @@ package snapshot import ( "fmt" + "net/http" "k8s.io/client-go/kubernetes" @@ -34,10 +35,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" enterprisesv1 "github.com/upmc-enterprises/elasticsearch-operator/pkg/apis/elasticsearchoperator/v1" - batchv1 "k8s.io/api/batch/v1" - v2alpha1 "k8s.io/api/batch/v2alpha1" + v1beta1 "k8s.io/api/batch/v1beta1" apicore "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( @@ -91,18 +92,12 @@ func (s *Scheduler) Init() error { // CreateSnapshotRepository creates the snapshot repository cronjob func (s *Scheduler) CreateSnapshotRepository() error { // TODO: This should wait until the api goes green and cluster is healthy - if err := s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, CRON_ACTION_REPOSITORY, s.CRD.CronSchedule); err != nil { - return err - } - return nil + return s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, CRON_ACTION_REPOSITORY, s.CRD.CronSchedule) } // CreateSnapshot creates snapshot cronjob func (s *Scheduler) CreateSnapshot() error { - if err := s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, CRON_ACTION_SNAPSHOT, s.CRD.CronSchedule); err != nil { - return err - } - return nil + return s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, CRON_ACTION_SNAPSHOT, s.CRD.CronSchedule) } // Stop cleans up Cron @@ -119,26 +114,43 @@ func (s *Scheduler) deleteJobs(namespace, clusterName string) { LabelSelector: fmt.Sprintf("app=elasticsearch-operator,clusterName=%s", clusterName), }) + // ignore not found error if err != nil { - logrus.Error("Could not delete Jobs! ", err) + if err.(*apierrors.StatusError).ErrStatus.Code != http.StatusNotFound { + logrus.Error("Could not delete Jobs! ", err) + } } + } // DeleteCronJob deletes a cron job func (s *Scheduler) deleteCronJob(namespace, clusterName string) { // Repository CronJob snapshotName := getSnapshotname(clusterName, CRON_ACTION_REPOSITORY) - err := s.Kclient.BatchV2alpha1().CronJobs(namespace).Delete(snapshotName, &metav1.DeleteOptions{}) + err := s.Kclient.BatchV1beta1().CronJobs(namespace).Delete(snapshotName, &metav1.DeleteOptions{}) + + // ignore not found error if err != nil { - logrus.Error("Could not delete Repository CronJob! ", err) + if _, ok := err.(*apierrors.StatusError); ok { + if err.(*apierrors.StatusError).ErrStatus.Code != http.StatusNotFound { + logrus.Error("Could not delete Repository CronJob! ", err) + } + } } // Snapshot CronJob snapshotName = getSnapshotname(clusterName, CRON_ACTION_SNAPSHOT) - err = s.Kclient.BatchV2alpha1().CronJobs(namespace).Delete(snapshotName, &metav1.DeleteOptions{}) + err = s.Kclient.BatchV1beta1().CronJobs(namespace).Delete(snapshotName, &metav1.DeleteOptions{}) + + // ignore not found error if err != nil { - logrus.Error("Could not delete CronJob! ", err) + if _, ok := err.(*apierrors.StatusError); ok { + if err.(*apierrors.StatusError).ErrStatus.Code != http.StatusNotFound { + logrus.Error("Could not delete CronJob! ", err) + } + } } + } // CreateCronJob creates a cron job @@ -146,7 +158,7 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s snapshotName := getSnapshotname(clusterName, action) // Check if CronJob exists - cronJob, err := s.Kclient.BatchV2alpha1().CronJobs(namespace).Get(snapshotName, metav1.GetOptions{}) + cronJob, err := s.Kclient.BatchV1beta1().CronJobs(namespace).Get(snapshotName, metav1.GetOptions{}) if len(cronJob.Name) == 0 { @@ -159,7 +171,7 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s if err == nil { return err } - job := &v2alpha1.CronJob{ + job := &v1beta1.CronJob{ ObjectMeta: metav1.ObjectMeta{ Name: snapshotName, Labels: map[string]string{ @@ -168,9 +180,9 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s "name": snapshotName, }, }, - Spec: v2alpha1.CronJobSpec{ + Spec: v1beta1.CronJobSpec{ Schedule: cronSchedule, - JobTemplate: v2alpha1.JobTemplateSpec{ + JobTemplate: v1beta1.JobTemplateSpec{ Spec: batchv1.JobSpec{ Template: apicore.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -209,7 +221,7 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s }, } - if _, err := s.Kclient.BatchV2alpha1().CronJobs(namespace).Create(job); err != nil { + if _, err := s.Kclient.BatchV1beta1().CronJobs(namespace).Create(job); err != nil { logrus.Error("Could not create CronJob! ", err) return err } @@ -217,6 +229,8 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s logrus.Error("Could not get cron job! ", err) return err } + logrus.Infof("CronJob %v succesfully created ! ", snapshotName) + return nil } From e24dd651b9d1bcd816b59359c824dde01beccae8 Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Thu, 5 Apr 2018 14:37:51 +0200 Subject: [PATCH 2/3] Add ability to customize cronjob image --- example/example-es-cluster-azure.yaml | 1 + example/example-es-cluster-hostpath.yaml | 1 + example/example-es-cluster-minikube.yaml | 1 + example/example-es-cluster-rook.yaml | 1 + example/example-es-cluster.yaml | 1 + pkg/apis/elasticsearchoperator/v1/cluster.go | 4 ++++ pkg/processor/processor.go | 1 + pkg/snapshot/scheduler.go | 24 ++++++++++++-------- 8 files changed, 25 insertions(+), 9 deletions(-) diff --git a/example/example-es-cluster-azure.yaml b/example/example-es-cluster-azure.yaml index f4d4b63c7..db05d5c0e 100644 --- a/example/example-es-cluster-azure.yaml +++ b/example/example-es-cluster-azure.yaml @@ -15,6 +15,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" + image: upmcenterprises/elasticsearch-cron:0.0.1 storage: storage-class: default resources: diff --git a/example/example-es-cluster-hostpath.yaml b/example/example-es-cluster-hostpath.yaml index 5529a93a4..2bc8bba86 100644 --- a/example/example-es-cluster-hostpath.yaml +++ b/example/example-es-cluster-hostpath.yaml @@ -19,5 +19,6 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" + image: upmcenterprises/elasticsearch-cron:0.0.1 storage: storage-class: hostpath-storage diff --git a/example/example-es-cluster-minikube.yaml b/example/example-es-cluster-minikube.yaml index 116071abe..78305e4b7 100644 --- a/example/example-es-cluster-minikube.yaml +++ b/example/example-es-cluster-minikube.yaml @@ -19,6 +19,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" + image: upmcenterprises/elasticsearch-cron:0.0.1 storage: type: standard storage-class-version: volume.alpha.kubernetes.io/storage-class diff --git a/example/example-es-cluster-rook.yaml b/example/example-es-cluster-rook.yaml index 78f5a6ea6..4d5dfcc5b 100644 --- a/example/example-es-cluster-rook.yaml +++ b/example/example-es-cluster-rook.yaml @@ -17,6 +17,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" + image: upmcenterprises/elasticsearch-cron:0.0.1 storage: storage-class: rook-block resources: diff --git a/example/example-es-cluster.yaml b/example/example-es-cluster.yaml index 2c4664b38..5851e782e 100644 --- a/example/example-es-cluster.yaml +++ b/example/example-es-cluster.yaml @@ -20,6 +20,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" + image: upmcenterprises/elasticsearch-cron:0.0.1 storage: type: gp2 storage-class-provisioner: kubernetes.io/aws-ebs diff --git a/pkg/apis/elasticsearchoperator/v1/cluster.go b/pkg/apis/elasticsearchoperator/v1/cluster.go index 151b751b7..a4fcf6923 100644 --- a/pkg/apis/elasticsearchoperator/v1/cluster.go +++ b/pkg/apis/elasticsearchoperator/v1/cluster.go @@ -142,6 +142,9 @@ type Snapshot struct { // Authentication defines credentials for snapshot requests Authentication Authentication `json:"authentication"` + + // Defines the image to run cronjobs + Image string `json:"image"` } // Authentication defines credentials for snapshot requests @@ -208,6 +211,7 @@ type Scheduler struct { ElasticURL string Namespace string ClusterName string + Image string } // SchedulerAuthentication stores credentials used to authenticate against snapshot endpoint diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 75bbc3921..53c7e1c06 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -214,6 +214,7 @@ func (p *Processor) refreshClusters() error { cluster.Spec.Snapshot.SchedulerEnabled, cluster.Spec.Snapshot.Authentication.UserName, cluster.Spec.Snapshot.Authentication.Password, + cluster.Spec.Snapshot.Image, p.k8sclient.GetClientServiceNameFullDNS(cluster.ObjectMeta.Name, cluster.ObjectMeta.Namespace), cluster.ObjectMeta.Name, cluster.ObjectMeta.Namespace, diff --git a/pkg/snapshot/scheduler.go b/pkg/snapshot/scheduler.go index bb6f01673..ff0f2d396 100644 --- a/pkg/snapshot/scheduler.go +++ b/pkg/snapshot/scheduler.go @@ -42,9 +42,9 @@ import ( ) const ( - baseCronImage = "upmcenterprises/elasticsearch-cron:0.0.1" - CRON_ACTION_REPOSITORY = "create-repository" - CRON_ACTION_SNAPSHOT = "snapshot" + defaultCronImage = "upmcenterprises/elasticsearch-cron:0.0.1" + cronActionRepository = "create-repository" + cronActionSnapshot = "snapshot" ) type Scheduler struct { @@ -53,7 +53,12 @@ type Scheduler struct { } // New creates an instance of Scheduler -func New(bucketName, cronSchedule string, enabled bool, userName, password, elasticURL, clusterName, namespace string, kc kubernetes.Interface) *Scheduler { +func New(bucketName, cronSchedule string, enabled bool, userName, password, image, + elasticURL, clusterName, namespace string, kc kubernetes.Interface) *Scheduler { + + if image == "" { + image = defaultCronImage + } return &Scheduler{ Kclient: kc, @@ -68,6 +73,7 @@ func New(bucketName, cronSchedule string, enabled bool, userName, password, elas Namespace: namespace, ClusterName: clusterName, Enabled: enabled, + Image: image, }, } } @@ -92,12 +98,12 @@ func (s *Scheduler) Init() error { // CreateSnapshotRepository creates the snapshot repository cronjob func (s *Scheduler) CreateSnapshotRepository() error { // TODO: This should wait until the api goes green and cluster is healthy - return s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, CRON_ACTION_REPOSITORY, s.CRD.CronSchedule) + return s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, cronActionRepository, s.CRD.CronSchedule) } // CreateSnapshot creates snapshot cronjob func (s *Scheduler) CreateSnapshot() error { - return s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, CRON_ACTION_SNAPSHOT, s.CRD.CronSchedule) + return s.CreateCronJob(s.CRD.Namespace, s.CRD.ClusterName, cronActionSnapshot, s.CRD.CronSchedule) } // Stop cleans up Cron @@ -126,7 +132,7 @@ func (s *Scheduler) deleteJobs(namespace, clusterName string) { // DeleteCronJob deletes a cron job func (s *Scheduler) deleteCronJob(namespace, clusterName string) { // Repository CronJob - snapshotName := getSnapshotname(clusterName, CRON_ACTION_REPOSITORY) + snapshotName := getSnapshotname(clusterName, cronActionRepository) err := s.Kclient.BatchV1beta1().CronJobs(namespace).Delete(snapshotName, &metav1.DeleteOptions{}) // ignore not found error @@ -139,7 +145,7 @@ func (s *Scheduler) deleteCronJob(namespace, clusterName string) { } // Snapshot CronJob - snapshotName = getSnapshotname(clusterName, CRON_ACTION_SNAPSHOT) + snapshotName = getSnapshotname(clusterName, cronActionSnapshot) err = s.Kclient.BatchV1beta1().CronJobs(namespace).Delete(snapshotName, &metav1.DeleteOptions{}) // ignore not found error @@ -197,7 +203,7 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s Containers: []apicore.Container{ apicore.Container{ Name: snapshotName, - Image: baseCronImage, + Image: s.CRD.Image, ImagePullPolicy: "Always", Resources: apicore.ResourceRequirements{ Requests: apicore.ResourceList{ From f365f37c500d96f7342f49d9287fae1b0cd9a63e Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Fri, 27 Apr 2018 14:53:29 +0200 Subject: [PATCH 3/3] Bump elasticsearch cron image to v0.3 --- example/example-es-cluster-azure.yaml | 2 +- example/example-es-cluster-hostpath.yaml | 2 +- example/example-es-cluster-minikube.yaml | 2 +- example/example-es-cluster-rook.yaml | 2 +- example/example-es-cluster.yaml | 2 +- pkg/snapshot/scheduler.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/example/example-es-cluster-azure.yaml b/example/example-es-cluster-azure.yaml index db05d5c0e..de5c5dd28 100644 --- a/example/example-es-cluster-azure.yaml +++ b/example/example-es-cluster-azure.yaml @@ -15,7 +15,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" - image: upmcenterprises/elasticsearch-cron:0.0.1 + image: upmcenterprises/elasticsearch-cron:0.0.3 storage: storage-class: default resources: diff --git a/example/example-es-cluster-hostpath.yaml b/example/example-es-cluster-hostpath.yaml index 2bc8bba86..4cc780f2c 100644 --- a/example/example-es-cluster-hostpath.yaml +++ b/example/example-es-cluster-hostpath.yaml @@ -19,6 +19,6 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" - image: upmcenterprises/elasticsearch-cron:0.0.1 + image: upmcenterprises/elasticsearch-cron:0.0.3 storage: storage-class: hostpath-storage diff --git a/example/example-es-cluster-minikube.yaml b/example/example-es-cluster-minikube.yaml index 78305e4b7..a81634fcc 100644 --- a/example/example-es-cluster-minikube.yaml +++ b/example/example-es-cluster-minikube.yaml @@ -19,7 +19,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" - image: upmcenterprises/elasticsearch-cron:0.0.1 + image: upmcenterprises/elasticsearch-cron:0.0.3 storage: type: standard storage-class-version: volume.alpha.kubernetes.io/storage-class diff --git a/example/example-es-cluster-rook.yaml b/example/example-es-cluster-rook.yaml index 4d5dfcc5b..e9b81e0c6 100644 --- a/example/example-es-cluster-rook.yaml +++ b/example/example-es-cluster-rook.yaml @@ -17,7 +17,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" - image: upmcenterprises/elasticsearch-cron:0.0.1 + image: upmcenterprises/elasticsearch-cron:0.0.3 storage: storage-class: rook-block resources: diff --git a/example/example-es-cluster.yaml b/example/example-es-cluster.yaml index 5851e782e..f3276f02d 100644 --- a/example/example-es-cluster.yaml +++ b/example/example-es-cluster.yaml @@ -20,7 +20,7 @@ spec: scheduler-enabled: false bucket-name: elasticsnapshots99 cron-schedule: "@every 2m" - image: upmcenterprises/elasticsearch-cron:0.0.1 + image: upmcenterprises/elasticsearch-cron:0.0.3 storage: type: gp2 storage-class-provisioner: kubernetes.io/aws-ebs diff --git a/pkg/snapshot/scheduler.go b/pkg/snapshot/scheduler.go index ff0f2d396..5e7dddf2f 100644 --- a/pkg/snapshot/scheduler.go +++ b/pkg/snapshot/scheduler.go @@ -42,7 +42,7 @@ import ( ) const ( - defaultCronImage = "upmcenterprises/elasticsearch-cron:0.0.1" + defaultCronImage = "upmcenterprises/elasticsearch-cron:0.0.3" cronActionRepository = "create-repository" cronActionSnapshot = "snapshot" )