From 34acbaacb593907b4371976ceddc84e1a87a885c Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Fri, 16 Nov 2018 11:10:32 +0530 Subject: [PATCH 01/18] doc change --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a45697ed7..33b74791c 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,8 @@ $ go get -u github.com/cloudflare/cfssl/cmd/cfssl $ go get -u github.com/cloudflare/cfssl/cmd/cfssljson $ go run cmd/operator/main.go --kubecfg-file=${HOME}/.kube/config ``` +#Scaling feature: + # About Built by UPMC Enterprises in Pittsburgh, PA. http://enterprises.upmc.com/ From a51e12277346d1957a8b853b390927daa97ffe48 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Fri, 16 Nov 2018 11:18:11 +0530 Subject: [PATCH 02/18] doc changes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33b74791c..06d18d5c8 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ $ go get -u github.com/cloudflare/cfssl/cmd/cfssl $ go get -u github.com/cloudflare/cfssl/cmd/cfssljson $ go run cmd/operator/main.go --kubecfg-file=${HOME}/.kube/config ``` -#Scaling feature: +# Scaling feature: # About From 751dd4575af888de3642016f0740b64d396ecaca Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Tue, 20 Nov 2018 23:41:16 +0530 Subject: [PATCH 03/18] changes: scaling feature changes, code clean up and stress test is pending. --- README.md | 45 +++++ cmd/operator/main.go | 16 +- pkg/apis/elasticsearchoperator/v1/cluster.go | 14 +- pkg/k8sutil/deployments.go | 16 +- pkg/k8sutil/es_crud.go | 201 +++++++++++++++++++ pkg/k8sutil/k8sutil.go | 33 ++- pkg/k8sutil/scaling.go | 121 +++++++++++ pkg/processor/processor.go | 76 +++++-- 8 files changed, 478 insertions(+), 44 deletions(-) create mode 100644 pkg/k8sutil/es_crud.go create mode 100644 pkg/k8sutil/scaling.go diff --git a/README.md b/README.md index 06d18d5c8..5fd5e8f85 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,51 @@ $ go get -u github.com/cloudflare/cfssl/cmd/cfssljson $ go run cmd/operator/main.go --kubecfg-file=${HOME}/.kube/config ``` # Scaling feature: +The following are changes present in scaling patch + +- Every Data node will have stateful set: currently all data nodes are part of one Statefule set, scaling needs every data node to be seperate statefulset with one replica, every datanode resource are updated independently to corresponding statefulset, so that the control will be in the hands of operator instead of k8. +- Scaling is optional feature: Seperate section is defined for scaling as shown in below example spec. If the scaling section is not present then entire scaling feature will be disabled. +- when Scaling will be triggered: Scaling will be triggered if there is any change in the one of the following 3 fields inside the scaling section. javaoptions and resources entries corresponds only to non-data nodes incase scaling section is present. If scaling section is abscent then it corresponds to all nodes. + - JavaOptions: This is the new field present inside the scaling section corresponds only to Data nodes. + - CPU inside resources : number of cpu cores. + - Memory inside resources : Memory size. + - Steps involved in vertical scaling of Elastic cluster: Repeating the following steps for each data node one after another, if there is any failure rest of scaling will be halted. + - Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory. + - Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. + - Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. + - Step-4: scale the Data node by updating the new resources in the stateful set + - Step-5: check if the POD is restarted and in running state from k8 point of view. + - Step-6: check if the POD is up from the ES point of view + - Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. + - Step-8: Undo the timeout settings. + - Future Enhancements: + - Horizontal scaling of Data nodes: increasing the "data-node-replica" will function as expected, but if "data-node-replica" is decreased by more then 2 then the Elastic search cluster can enter in to red state and there will be data loss, this can prevented by executing similar to vertical scaling one after another. + + +``` +Example Spec containing optional scaling section + +spec: + client-node-replicas: 3 + data-node-replicas: 3 + data-volume-size: 10Gi + java-options: -Xms256m -Xmx256m + master-node-replicas: 2 + scaling: + java-options: -Xms1052m -Xmx1052m + resources: + limits: + cpu: 2m + memory: 2048Mi + requests: + cpu: 1m + memory: 1024Mi + zones: + - us-east-1a + - us-east-1b + - us-east-1c +``` + # About diff --git a/cmd/operator/main.go b/cmd/operator/main.go index b2f4cd809..5ca5b63cd 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -35,9 +35,9 @@ import ( "syscall" "github.com/Sirupsen/logrus" - "github.com/heptiolabs/healthcheck" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" +// "github.com/heptiolabs/healthcheck" +// "github.com/prometheus/client_golang/prometheus" +// "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/upmc-enterprises/elasticsearch-operator/pkg/controller" "github.com/upmc-enterprises/elasticsearch-operator/pkg/k8sutil" @@ -102,17 +102,17 @@ func Main() int { doneChan := make(chan struct{}) var wg sync.WaitGroup - +/* JANA r := prometheus.NewRegistry() r.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) r.MustRegister(prometheus.NewGoCollector()) health := healthcheck.NewMetricsHandler(r, "elasticsearch-operator") - +*/ mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{})) - mux.HandleFunc("/live", health.LiveEndpoint) - mux.HandleFunc("/ready", health.ReadyEndpoint) +// mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{})) +// mux.HandleFunc("/live", health.LiveEndpoint) +// mux.HandleFunc("/ready", health.ReadyEndpoint) // Kick it off controller.Run() diff --git a/pkg/apis/elasticsearchoperator/v1/cluster.go b/pkg/apis/elasticsearchoperator/v1/cluster.go index b33303226..4d2acc0a8 100644 --- a/pkg/apis/elasticsearchoperator/v1/cluster.go +++ b/pkg/apis/elasticsearchoperator/v1/cluster.go @@ -88,7 +88,10 @@ type ClusterSpec struct { // Snapshot defines how snapshots are scheduled Snapshot Snapshot `json:"snapshot"` - + + // Scaling defines how data nodes does virtual-scaling + Scaling Scaling `json:"scaling"` + // Storage defines how volumes are provisioned Storage Storage `json:"storage"` @@ -134,6 +137,15 @@ type ImagePullSecrets struct { Name string `json:"name"` } +// Scaling defines all params for vertical scaling of data nodes +type Scaling struct { + // Resources defines memory / cpu constraints + Resources Resources `json:"resources"` + + // JavaOptions defines args passed to elastic nodes + JavaOptions string `json:"java-options"` +} + // Snapshot defines all params to create / store snapshots type Snapshot struct { // Enabled determines if snapshots are enabled diff --git a/pkg/k8sutil/deployments.go b/pkg/k8sutil/deployments.go index f50fd7250..b6e6755e1 100644 --- a/pkg/k8sutil/deployments.go +++ b/pkg/k8sutil/deployments.go @@ -122,11 +122,11 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java limitMemory, _ := resource.ParseQuantity(resources.Limits.Memory) requestCPU, _ := resource.ParseQuantity(resources.Requests.CPU) requestMemory, _ := resource.ParseQuantity(resources.Requests.Memory) - scheme := v1.URISchemeHTTP +// scheme := v1.URISchemeHTTP if useSSL != nil && *useSSL { - scheme = v1.URISchemeHTTPS +// scheme = v1.URISchemeHTTPS } - probe := &v1.Probe{ + /*probe := &v1.Probe{ TimeoutSeconds: 30, InitialDelaySeconds: 10, FailureThreshold: 15, @@ -137,7 +137,7 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java Scheme: scheme, }, }, - } + }*/ deployment := &v1beta1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, @@ -194,6 +194,10 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java Name: "NODE_DATA", Value: "false", }, + v1.EnvVar{ + Name: "NODE_INGEST", + Value: "true", + }, v1.EnvVar{ Name: "HTTP_ENABLE", Value: "true", @@ -235,8 +239,8 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java Protocol: v1.ProtocolTCP, }, }, - ReadinessProbe: probe, - LivenessProbe: probe, + //ReadinessProbe: probe, + //LivenessProbe: probe, VolumeMounts: []v1.VolumeMount{ v1.VolumeMount{ Name: "storage", diff --git a/pkg/k8sutil/es_crud.go b/pkg/k8sutil/es_crud.go new file mode 100644 index 000000000..fc90abc0d --- /dev/null +++ b/pkg/k8sutil/es_crud.go @@ -0,0 +1,201 @@ + + +package k8sutil + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" +) + + +/* TODO-1 : This speeds up scaling. +"persistent" : { + "cluster.routing.allocation.node_concurrent_recoveries": 20, from 2 to 20 + } + +TODO-2: + set translog durabilty to request: index.translog.durability + +After scaling operation it can be converted to async. +*/ +func es_unset_delaytimeout(es_ip string)(bool){ /* TODO : need to undio this setting at the end */ + ret := false + return ret +} +func es_set_delaytimeout(es_ip string)(bool){ /* TODO : need to undio this setting at the end */ + ret := false + // return true + + client := &http.Client{ + } + body:="{ " + body = body + "\"settings\": { \"index.unassigned.node_left.delayed_timeout\": \"6m\" }" + body = body + " }" + + //fmt.Println("set BODY :",body,":") + req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_all/_settings", bytes.NewBufferString(body)) + req.Header.Add("Content-Type", `application/json`) + resp, _ := client.Do(req) + data, _ := ioutil.ReadAll(resp.Body) + + + if (resp.StatusCode == 200){ + ret = true + }else{ + if (strings.Contains(string(data), "index_not_found_exception")){ + ret = true /* if there are no indexes , then this function need tpo pass */ + fmt.Println("WARNING in setting delaytimeout: response: ",resp, string(data)) + return ret + } + fmt.Println("ERROR in setting delaytimeout: response: ",resp, string(data)) + } + return ret; +} +func es_checkForGreen(nodename string)(error){ /* TODO */ + return nil +} +func WordCount(input string, nodename string) (int,int) { + node_count := 0 + unassigned_count :=0 + initializing_count :=0 + words := strings.Fields(input) + for _, word := range words { + if (word == nodename){ + node_count++ + } else if (word == "UNASSIGNED"){ + unassigned_count++ + } + if (word == "INITIALIZING"){ + initializing_count++ + } + } + fmt.Println(nodename," shards: ",node_count," UNASSIGNED shards: ",unassigned_count," initialising shards: ",initializing_count) + return node_count,unassigned_count+initializing_count +} + +func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { + var ret error + ret = errors.New("still unassigned shards are there") + + for i := 0; i < waitSeconds; i++ { + response, err := http.Get("http://" + es_ip + "/_cat/shards") + if err != nil { + fmt.Printf("The HTTP request failed with error %s\n", err) + } else { + data, _ := ioutil.ReadAll(response.Body) + _,unassigned_count := WordCount(string(data), nodeName) + if (unassigned_count>0) { + time.Sleep(1 * time.Second) + continue + } else { + ret = nil + break + } + } + time.Sleep(1 * time.Second) + } + + return ret; +} + + func es_checkForNodeUp(es_ip string, nodeName string, waitSeconds int)(error){ + var ret error + ret = errors.New("ES node is not up") + + for i := 0; i < waitSeconds; i++ { + response, err := http.Get("http://" + es_ip + "/_cat/nodes?v&h=n,ip,v") + if err != nil { + fmt.Printf("The HTTP request failed with error %s\n", err) + ret = err + } else { + data, _ := ioutil.ReadAll(response.Body) + str_ret := strings.Contains(string(data), nodeName) + if (str_ret){ + //time.Sleep(1 * time.Second) + //fmt.Println("http output: nodename: ",nodeName," DATA: ", string(data)) + ret = nil + return ret; + } + } + time.Sleep(1 * time.Second) + } + + return ret; + } +/* +es_flush: response: &{409 Conflict 409 HTTP/1.1 1 1 map[Content-Type:[application/json; charset=UTF-8]] 0xc420017a60 -1 [] false true map[] 0xc4200d9600 } BODY: {"_shards":{"total":32,"successful":31,"failed":1}, +"indexer_tenant_1":{"total":16,"successful":16,"failed":0},"indexer_tenant_0":{"total":16,"successful":15,"failed":1,"failures":[{"shard":4,"reason":"pending operations","routing":{"state":"STARTED","primary":false,"node":"pGFMewaYSG-acz3GHywEzQ","relocating_node":null,"shard":4,"index":"indexer_tenant_0","allocation_id":{"id":"fu9axLfdQZqbwBKXd4DSPw"}}}]}} + */ + +func es_flush(es_ip string)(bool){ + ret := false + + client := &http.Client{ + } + + req, _ := http.NewRequest("POST", "http://"+es_ip+"/_all/_flush/synced", bytes.NewBufferString("")) + req.Header.Add("Content-Type", `application/json`) + resp, _ := client.Do(req) + data, _ := ioutil.ReadAll(resp.Body) + fmt.Println("es_flush: response: ",resp," BODY: ",string(data)) + if (resp.StatusCode == 200){ + ret = true + time.Sleep(1 * time.Second) + } + return ret; +} + +func es_set_readonly(es_ip string)(bool){ + ret := false +// return true + + client := &http.Client{ + } + body:="{ " + //body= body +"{ \"persistent\": { \"cluster.blocks.read_only\": \"true\" } }" + body = body + "\"transient\": { \"cluster.blocks.read_only\": \"true\" }" +// body = body + ", \"cluster.routing.allocation.enable\": \"none\" } " + body = body + " }" + + //fmt.Println("set BODY :",body,":") + req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_cluster/settings", bytes.NewBufferString(body)) + req.Header.Add("Content-Type", `application/json`) + resp, _ := client.Do(req) + fmt.Println("es_flush: response: ",resp, resp.Body) + if (resp.StatusCode == 200){ + ret = true + time.Sleep(1 * time.Second) + } + es_flush(es_ip) + + return ret; +} +func es_unset_readonly(es_ip string)(bool){ + ret := false +// return true + + client := &http.Client{ + } + body:="{ " + //body= body +"{ \"persistent\": { \"cluster.blocks.read_only\": \"false\" } }" + body = body + "\"transient\": { \"cluster.blocks.read_only\": \"false\" }" + //body = body + ", \"cluster.routing.allocation.enable\": \"all\" } " + body = body + " }" + + fmt.Println("unset BODY :",body,":") + req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_cluster/settings", bytes.NewBufferString(body)) + req.Header.Add("Content-Type", `application/json`) + resp, _ := client.Do(req) + fmt.Println("es_unset: response: ",resp) + if (resp.StatusCode == 200){ + ret = true + time.Sleep(1 * time.Second) + } + + return ret; +} \ No newline at end of file diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index f1358969b..89bc7ea23 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -45,7 +45,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/intstr" + //"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -403,10 +403,10 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s volumeSize, _ := resource.ParseQuantity(dataDiskSize) enableSSL := "true" - scheme := v1.URISchemeHTTPS + // scheme := v1.URISchemeHTTPS if useSSL != nil && !*useSSL { enableSSL = "false" - scheme = v1.URISchemeHTTP + // scheme = v1.URISchemeHTTP } // Parse CPU / Memory @@ -415,7 +415,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s requestCPU, _ := resource.ParseQuantity(resources.Requests.CPU) requestMemory, _ := resource.ParseQuantity(resources.Requests.Memory) - readinessProbe := &v1.Probe{ + /*readinessProbe := &v1.Probe{ TimeoutSeconds: 30, InitialDelaySeconds: 10, FailureThreshold: 15, @@ -437,7 +437,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s Scheme: scheme, }, }, - } + }*/ component := fmt.Sprintf("elasticsearch-%s", clusterName) discoveryServiceNameCluster := fmt.Sprintf("%s-%s", discoveryServiceName, clusterName) @@ -519,6 +519,14 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s Name: "CLUSTER_NAME", Value: clusterName, }, + v1.EnvVar{ + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, v1.EnvVar{ Name: "NODE_MASTER", Value: isNodeMaster, @@ -568,8 +576,8 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s Protocol: v1.ProtocolTCP, }, }, - ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe, + /*ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe,*/ VolumeMounts: []v1.VolumeMount{ v1.VolumeMount{ Name: "es-data", @@ -653,11 +661,11 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s // CreateDataNodeDeployment creates the data node deployment func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources, - imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string) error { + imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string, index int,scaling bool) error { deploymentName, _, _, _ := processDeploymentType(deploymentType, clusterName) - statefulSetName := fmt.Sprintf("%s-%s", deploymentName, storageClass) + statefulSetName := fmt.Sprintf("%s-%s-%d", deploymentName, storageClass, index) // Check if StatefulSet exists statefulSet, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) @@ -678,10 +686,13 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3 logrus.Error("Could not get stateful set! ", err) return err } - + if deploymentType == "data" && scaling { + return scale_datanode(k, namespace, statefulSetName, resources, javaOptions, statefulSet) + } //scale replicas? if statefulSet.Spec.Replicas != replicas { currentReplicas := *statefulSet.Spec.Replicas + logrus.Infof(" Current replicas: ",currentReplicas," New replica: ",*replicas) if *replicas < currentReplicas { minMasterNodes := elasticsearchutil.MinMasterNodes(int(*replicas)) logrus.Infof("Detected master scale-down. Setting 'discovery.zen.minimum_master_nodes' to %d", minMasterNodes) @@ -689,7 +700,7 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3 } statefulSet.Spec.Replicas = replicas _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Update(statefulSet) - + logrus.Infof(" updated Stateful set: ", statefulSetName) if err != nil { logrus.Error("Could not scale statefulSet: ", err) minMasterNodes := elasticsearchutil.MinMasterNodes(int(currentReplicas)) diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go new file mode 100644 index 000000000..d2b747f9f --- /dev/null +++ b/pkg/k8sutil/scaling.go @@ -0,0 +1,121 @@ +/* + + */ + +package k8sutil +import ( + "fmt" + "github.com/Sirupsen/logrus" + myspec "github.com/upmc-enterprises/elasticsearch-operator/pkg/apis/elasticsearchoperator/v1" + "k8s.io/api/apps/v1beta2" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" + "strconv" +) + +func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) error { + ret := fmt.Errorf("Scaling POD is not up") + dnodename := statefulSetName + "-0" + newstatefulset, _ := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) + //logrus.Infof("Scaling: statefulset : %v ", newstatefulset) + + ss_version , _ := strconv.Atoi(newstatefulset.ObjectMeta.ResourceVersion) + + pod, _ := k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) + logrus.Infof("Scaling NEW POD ",pod.ObjectMeta.ResourceVersion) + waitSeconds := 400 + for i := 0; i < waitSeconds; i++ { + pod, _ = k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) + if (pod == nil){ + logrus.Infof("Scaling POD Terminated ") + }else{ + pod_version , _ := strconv.Atoi(pod.ObjectMeta.ResourceVersion) + if pod_version > ss_version { + logrus.Infof("Scaling POD started pod_version:",pod_version," dd_version:",ss_version," state: ",pod.Status.ContainerStatuses) + if strings.Contains(dn.runningStatus,"ContainerStateRunning") { + ret = nil + break + } + }else{ + logrus.Infof("Scaling pod revision: ", pod.ObjectMeta.ResourceVersion, " ss revision: ", newstatefulset.ObjectMeta.ResourceVersion) + } + } + time.Sleep(5 * time.Second) + } + return ret +} + +func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet) error { + cpu, _ := resource.ParseQuantity(resources.Requests.CPU) + memory, _ := resource.ParseQuantity(resources.Requests.Memory) + +// Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory + resourceMatch := true + if cpu != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] { + logrus.Infof("CPU Changed USER: ", cpu, " from k8: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) + resourceMatch = false + statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpu + } + if memory != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] { + // TODO: there is a slight mismatch, Memory Changed USER: %!(EXTRA resource.Quantity={{1073741824 0} {} BinarySI}, string= from k8: , resource.Quantity={{1073741824 0} {} 1Gi BinarySI}) + //logrus.Infof(" Memory Changed USER: ",memory," from k8: ",statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"].i.value) + //resourceMatch = false + } + for index, env := range statefulSet.Spec.Template.Spec.Containers[0].Env { + if env.Name == "ES_JAVA_OPTS" { + if env.Value != javaOptions { + logrus.Infof(" JAVA OPTS Changed USER: ", javaOptions, " from k8: ", env.Value) + resourceMatch = false + statefulSet.Spec.Template.Spec.Containers[0].Env[index].Value = javaOptions + } + break + } + } + + if !resourceMatch { + logrus.Infof(" RESOURCES CHANGED: updating Stateful set: ", statefulSetName) + // TODO : only memory request is updated, memory limit as need to be updated. + statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memory + + masterip := "worked9d5000000.lab.es-c1.eastus2.us.azure.k8s.walmart.com:30004" // TODO: need to remove later + + // Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. + if !es_set_delaytimeout(masterip) { + err := fmt.Errorf("Setting delaytimeout failed") + return err + } + // Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. + if err := es_checkForGreen(masterip); err != nil { // TODO : check green not implemented + err = fmt.Errorf("ES cluster is not in green state") + return err + } + + // Step-4: scale the Data node by updating the new resources in the stateful set + _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Update(statefulSet) + if err != nil { + logrus.Error("ERROR: Could not scale statefulSet: ", err) + return err + } + dnodename := statefulSetName + "-0" + + // Step-5: check if the POD is restarted and in running state from k8 point of view. + k8_check_DataNodeRestarted(namespace, statefulSetName, k) + + // Step-6: check if the POD is up from the ES point of view. + if err = es_checkForNodeUp(masterip, dnodename, 180); err != nil { + return err + } + logrus.Infof("Datanode Up in K8: ", dnodename) + // Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. + if err = es_checkForShards(masterip, dnodename, 180); err != nil { + return err + } + // Step-8: Undo the timeout settings + if !es_unset_delaytimeout(masterip) { + err = fmt.Errorf("Setting delaytimeout failed") + return err + } + } + return nil +} diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index af09b7c07..9a30992b0 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -27,7 +27,7 @@ package processor import ( "fmt" "sync" - + //"runtime/debug" "github.com/upmc-enterprises/elasticsearch-operator/pkg/elasticsearchutil" "github.com/upmc-enterprises/elasticsearch-operator/pkg/snapshot" @@ -151,7 +151,7 @@ func (p *Processor) refreshClusters() error { } for _, cluster := range currentClusters.Items { - logrus.Infof("Found cluster: %s", cluster.ObjectMeta.Name) + logrus.Infof("jana Found cluster: %s", cluster.ObjectMeta.Name) useSSL := p.defaultUseSSL(cluster.Spec.UseSSL) p.clusters[fmt.Sprintf("%s-%s", cluster.ObjectMeta.Name, cluster.ObjectMeta.Namespace)] = Cluster{ ESCluster: &myspec.ElasticsearchCluster{ @@ -171,6 +171,19 @@ func (p *Processor) refreshClusters() error { CronSchedule: cluster.Spec.Snapshot.CronSchedule, RepoRegion: cluster.Spec.Snapshot.RepoRegion, }, + Scaling: myspec.Scaling{ + JavaOptions: cluster.Spec.Scaling.JavaOptions, + Resources: myspec.Resources{ + Limits: myspec.MemoryCPU{ + Memory: cluster.Spec.Scaling.Resources.Limits.Memory, + CPU: cluster.Spec.Scaling.Resources.Limits.CPU, + }, + Requests: myspec.MemoryCPU{ + Memory: cluster.Spec.Scaling.Resources.Requests.Memory, + CPU: cluster.Spec.Scaling.Resources.Requests.CPU, + }, + }, + }, Storage: myspec.Storage{ StorageType: cluster.Spec.Storage.StorageType, StorageClassProvisoner: cluster.Spec.Storage.StorageClassProvisoner, @@ -320,8 +333,8 @@ func (p *Processor) processMasterPodEvent(c *v1.Pod) error { } func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) error { - logrus.Println("--------> Received ElasticSearch Event!") - + logrus.Println("--------> Received ElasticSearch Event!:", c) + //debug.PrintStack() // Refresh if err := p.refreshClusters(); err != nil { logrus.Error("Error refreshing cluster ", err) @@ -331,7 +344,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Is a base image defined in the custom cluster? var baseImage = p.calcBaseImage(p.baseImage, c.Spec.ElasticSearchImage) - logrus.Infof("Using [%s] as image for es cluster", baseImage) + logrus.Infof("JANA Using [%s] as image for es cluster", baseImage) // Default UseSSL to true useSSL := p.defaultUseSSL(c.Spec.UseSSL) @@ -364,13 +377,13 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) logrus.Error("Error creating client service ", err) return err } - + if err := p.k8sclient.CreateClientDeployment(baseImage, &c.Spec.ClientNodeReplicas, c.Spec.JavaOptions, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.UseSSL); err != nil { logrus.Error("Error creating client deployment ", err) return err } - + zoneCount := 0 if len(c.Spec.Zones) != 0 { zoneCount = len(c.Spec.Zones) @@ -390,17 +403,29 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionMaster { if err := p.k8sclient.CreateDataNodeDeployment("master", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL); err != nil { + c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,0 , false); err != nil { logrus.Error("Error creating master node deployment ", err) return err } } // Create Data Nodes + javaOptions := c.Spec.JavaOptions + resources := c.Spec.Resources + scaling := false + if len(c.Spec.Scaling.JavaOptions) != 0 { + javaOptions = c.Spec.Scaling.JavaOptions + scaling = true + } + if len(c.Spec.Scaling.Resources.Requests.CPU) != 0 { + resources = c.Spec.Scaling.Resources + scaling = true + } + logrus.Info("Scaling enabled: ",scaling," zone distribution JavaOptions:",javaOptions," Resources: ",resources) for index, count := range zoneDistributionData { - if err := p.k8sclient.CreateDataNodeDeployment("data", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, + if err := p.k8sclient.CreateDataNodeDeployment("data", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL); err != nil { + c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,index,scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err @@ -416,21 +441,36 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Create Master Nodes if err := p.k8sclient.CreateDataNodeDeployment("master", func() *int32 { i := int32(c.Spec.MasterNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, 0, false); err != nil { logrus.Error("Error creating master node deployment ", err) return err } // Create Data Nodes - if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, - c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL); err != nil { - logrus.Error("Error creating data node deployment ", err) - return err + dataNodeCount := int(c.Spec.DataNodeReplicas) /* TODO: if the replica count is decreased, then need to handle accordingly */ + javaOptions := c.Spec.JavaOptions + resources := c.Spec.Resources + scaling := false + if len(c.Spec.Scaling.JavaOptions) != 0 { + javaOptions = c.Spec.Scaling.JavaOptions + scaling = true + } + if len(c.Spec.Scaling.Resources.Requests.CPU) != 0 { + resources = c.Spec.Scaling.Resources + scaling = true + } + logrus.Info("Scaling enabled: ",scaling," JavaOptions:",javaOptions," Resources: ",resources," ESURL:",c.Spec.Scheduler.ElasticURL) + for nodeindex := 0; nodeindex < dataNodeCount; nodeindex++ { + if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(1); return &i }(), baseImage, c.Spec.Storage.StorageClass, + c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, nodeindex , scaling); err != nil { + logrus.Error("Error creating data node deployment ", err) + return err + } } } - + // Deploy Kibana if c.Spec.Kibana.Image != "" { @@ -475,7 +515,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) } } - + // Setup CronSchedule p.clusters[fmt.Sprintf("%s-%s", c.ObjectMeta.Name, c.ObjectMeta.Namespace)].Scheduler.Init() logrus.Println("--------> ElasticSearch Event finished!") From b57c50281538b3496f5cc23a196802c89fcab85c Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Wed, 21 Nov 2018 00:11:37 +0530 Subject: [PATCH 04/18] code cleanup. --- pkg/k8sutil/scaling.go | 24 ++++++++++++------------ pkg/processor/processor.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index d2b747f9f..e8aa20103 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -15,7 +15,7 @@ import ( ) func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) error { - ret := fmt.Errorf("Scaling POD is not up") + ret := fmt.Errorf("Scaling: POD is not up: ",statefulSetName) dnodename := statefulSetName + "-0" newstatefulset, _ := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) //logrus.Infof("Scaling: statefulset : %v ", newstatefulset) @@ -23,22 +23,22 @@ func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) e ss_version , _ := strconv.Atoi(newstatefulset.ObjectMeta.ResourceVersion) pod, _ := k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) - logrus.Infof("Scaling NEW POD ",pod.ObjectMeta.ResourceVersion) + logrus.Infof("Scaling: NEW POD ",pod.ObjectMeta.ResourceVersion) waitSeconds := 400 for i := 0; i < waitSeconds; i++ { pod, _ = k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) if (pod == nil){ - logrus.Infof("Scaling POD Terminated ") + logrus.Infof("Scaling: POD Terminated ") }else{ pod_version , _ := strconv.Atoi(pod.ObjectMeta.ResourceVersion) if pod_version > ss_version { - logrus.Infof("Scaling POD started pod_version:",pod_version," dd_version:",ss_version," state: ",pod.Status.ContainerStatuses) + logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",pod.Status.ContainerStatuses) if strings.Contains(dn.runningStatus,"ContainerStateRunning") { ret = nil break } }else{ - logrus.Infof("Scaling pod revision: ", pod.ObjectMeta.ResourceVersion, " ss revision: ", newstatefulset.ObjectMeta.ResourceVersion) + logrus.Infof("Scaling: pod revision: ", pod.ObjectMeta.ResourceVersion, " ss revision: ", newstatefulset.ObjectMeta.ResourceVersion) } } time.Sleep(5 * time.Second) @@ -53,7 +53,7 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys // Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory resourceMatch := true if cpu != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] { - logrus.Infof("CPU Changed USER: ", cpu, " from k8: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) + logrus.Infof("Scaling: step-1 : CPU Changed USER: ", cpu, " from k8: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) resourceMatch = false statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpu } @@ -65,7 +65,7 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys for index, env := range statefulSet.Spec.Template.Spec.Containers[0].Env { if env.Name == "ES_JAVA_OPTS" { if env.Value != javaOptions { - logrus.Infof(" JAVA OPTS Changed USER: ", javaOptions, " from k8: ", env.Value) + logrus.Infof("Scaling: step-1 JAVA OPTS Changed USER: ", javaOptions, " from k8: ", env.Value) resourceMatch = false statefulSet.Spec.Template.Spec.Containers[0].Env[index].Value = javaOptions } @@ -74,7 +74,7 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys } if !resourceMatch { - logrus.Infof(" RESOURCES CHANGED: updating Stateful set: ", statefulSetName) + logrus.Infof("Scaling: step-1 RESOURCES CHANGED: updating Stateful set: ", statefulSetName) // TODO : only memory request is updated, memory limit as need to be updated. statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memory @@ -82,19 +82,19 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys // Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. if !es_set_delaytimeout(masterip) { - err := fmt.Errorf("Setting delaytimeout failed") + err := fmt.Errorf("Scaling: step-2 Setting delaytimeout failed") return err } // Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. if err := es_checkForGreen(masterip); err != nil { // TODO : check green not implemented - err = fmt.Errorf("ES cluster is not in green state") + err = fmt.Errorf("Scaling: step-3 ES cluster is not in green state") return err } // Step-4: scale the Data node by updating the new resources in the stateful set _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Update(statefulSet) if err != nil { - logrus.Error("ERROR: Could not scale statefulSet: ", err) + logrus.Error("ERROR: Scaling: Could not scale statefulSet: ", err) return err } dnodename := statefulSetName + "-0" @@ -106,7 +106,7 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys if err = es_checkForNodeUp(masterip, dnodename, 180); err != nil { return err } - logrus.Infof("Datanode Up in K8: ", dnodename) + // Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. if err = es_checkForShards(masterip, dnodename, 180); err != nil { return err diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 9a30992b0..0acc64f12 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -151,7 +151,7 @@ func (p *Processor) refreshClusters() error { } for _, cluster := range currentClusters.Items { - logrus.Infof("jana Found cluster: %s", cluster.ObjectMeta.Name) + logrus.Infof("Found cluster: %s", cluster.ObjectMeta.Name) useSSL := p.defaultUseSSL(cluster.Spec.UseSSL) p.clusters[fmt.Sprintf("%s-%s", cluster.ObjectMeta.Name, cluster.ObjectMeta.Namespace)] = Cluster{ ESCluster: &myspec.ElasticsearchCluster{ From 46d133919036a24e95d86636989eb10834514623 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Thu, 22 Nov 2018 11:35:21 +0530 Subject: [PATCH 05/18] code cleanup --- pkg/k8sutil/es_crud.go | 38 ++++++++++---------- pkg/k8sutil/k8sutil.go | 82 ++++++++++++++++++++++++++---------------- pkg/k8sutil/scaling.go | 60 ++++++++++++++++++++----------- 3 files changed, 111 insertions(+), 69 deletions(-) diff --git a/pkg/k8sutil/es_crud.go b/pkg/k8sutil/es_crud.go index fc90abc0d..ca730dabc 100644 --- a/pkg/k8sutil/es_crud.go +++ b/pkg/k8sutil/es_crud.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" "time" + "github.com/Sirupsen/logrus" ) @@ -23,18 +24,16 @@ TODO-2: After scaling operation it can be converted to async. */ -func es_unset_delaytimeout(es_ip string)(bool){ /* TODO : need to undio this setting at the end */ - ret := false - return ret -} -func es_set_delaytimeout(es_ip string)(bool){ /* TODO : need to undio this setting at the end */ - ret := false - // return true +func es_set_delaytimeout(es_ip , duration string)(error){ + var ret error + ret = errors.New("Scaling: error in setting delay timeout") + + logrus.Infof("Scaling: setting ES delay timeout to ", duration) client := &http.Client{ } body:="{ " - body = body + "\"settings\": { \"index.unassigned.node_left.delayed_timeout\": \"6m\" }" + body = body + "\"settings\": { \"index.unassigned.node_left.delayed_timeout\": \""+duration+"\" }" body = body + " }" //fmt.Println("set BODY :",body,":") @@ -43,29 +42,30 @@ func es_set_delaytimeout(es_ip string)(bool){ /* TODO : need to undio this sett resp, _ := client.Do(req) data, _ := ioutil.ReadAll(resp.Body) - if (resp.StatusCode == 200){ - ret = true + ret = nil }else{ if (strings.Contains(string(data), "index_not_found_exception")){ - ret = true /* if there are no indexes , then this function need tpo pass */ + ret = nil /* if there are no indexes , then this function need po pass */ fmt.Println("WARNING in setting delaytimeout: response: ",resp, string(data)) return ret } - fmt.Println("ERROR in setting delaytimeout: response: ",resp, string(data)) + //string str:= "Scaling: setting delaytimeout: response: " + resp + string(data) + ret = errors.New("Scaling: setting delaytimeout: response: ") } return ret; } -func es_checkForGreen(nodename string)(error){ /* TODO */ - return nil +func es_checkForGreen(es_ip string)(error){ /* TODO */ + logrus.Infof("Scaling: ES checking for green ") + return es_checkForShards(es_ip , "", 180) } -func WordCount(input string, nodename string) (int,int) { +func util_wordcount(input string, nodename string) (int,int) { node_count := 0 unassigned_count :=0 initializing_count :=0 words := strings.Fields(input) for _, word := range words { - if (word == nodename){ + if (nodename != "" && word == nodename){ node_count++ } else if (word == "UNASSIGNED"){ unassigned_count++ @@ -81,14 +81,15 @@ func WordCount(input string, nodename string) (int,int) { func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { var ret error ret = errors.New("still unassigned shards are there") - + + logrus.Infof("Scaling: ES checking for shards") for i := 0; i < waitSeconds; i++ { response, err := http.Get("http://" + es_ip + "/_cat/shards") if err != nil { fmt.Printf("The HTTP request failed with error %s\n", err) } else { data, _ := ioutil.ReadAll(response.Body) - _,unassigned_count := WordCount(string(data), nodeName) + _,unassigned_count := util_wordcount(string(data), nodeName) if (unassigned_count>0) { time.Sleep(1 * time.Second) continue @@ -107,6 +108,7 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { var ret error ret = errors.New("ES node is not up") + logrus.Infof("Scaling: ES checking if the data node joined master") for i := 0; i < waitSeconds; i++ { response, err := http.Get("http://" + es_ip + "/_cat/nodes?v&h=n,ip,v") if err != nil { diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 89bc7ea23..90813c9e3 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -45,7 +45,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/errors" - //"k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -403,10 +403,10 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s volumeSize, _ := resource.ParseQuantity(dataDiskSize) enableSSL := "true" - // scheme := v1.URISchemeHTTPS + scheme := v1.URISchemeHTTPS if useSSL != nil && !*useSSL { enableSSL = "false" - // scheme = v1.URISchemeHTTP + scheme = v1.URISchemeHTTP } // Parse CPU / Memory @@ -415,7 +415,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s requestCPU, _ := resource.ParseQuantity(resources.Requests.CPU) requestMemory, _ := resource.ParseQuantity(resources.Requests.Memory) - /*readinessProbe := &v1.Probe{ + readinessProbe := &v1.Probe{ TimeoutSeconds: 30, InitialDelaySeconds: 10, FailureThreshold: 15, @@ -437,10 +437,52 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s Scheme: scheme, }, }, - }*/ + } component := fmt.Sprintf("elasticsearch-%s", clusterName) discoveryServiceNameCluster := fmt.Sprintf("%s-%s", discoveryServiceName, clusterName) + + volumes := []v1.Volume{} + volumeClaimTemplates := []v1.PersistentVolumeClaim{} + if storageClass == "localdisk" { + hostpath := "/mnt/"+statefulSetName + // hosttype := "Directory" + volumes = []v1.Volume{ + v1.Volume{ + Name: "es-data", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path : hostpath, + }, + }, + }, + + } + }else{ + volumeClaimTemplates = []v1.PersistentVolumeClaim{ + v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "es-data", + Labels: map[string]string{ + "component": "elasticsearch", + "role": role, + "name": statefulSetName, + "cluster": clusterName, + }, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: volumeSize, + }, + }, + }, + }, + } + } statefulSet := &apps.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -576,8 +618,8 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s Protocol: v1.ProtocolTCP, }, }, - /*ReadinessProbe: readinessProbe, - LivenessProbe: livenessProbe,*/ + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, VolumeMounts: []v1.VolumeMount{ v1.VolumeMount{ Name: "es-data", @@ -596,33 +638,11 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s }, }, }, - Volumes: []v1.Volume{}, + Volumes: volumes, ImagePullSecrets: TemplateImagePullSecrets(imagePullSecrets), }, }, - VolumeClaimTemplates: []v1.PersistentVolumeClaim{ - v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "es-data", - Labels: map[string]string{ - "component": "elasticsearch", - "role": role, - "name": statefulSetName, - "cluster": clusterName, - }, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - v1.ReadWriteOnce, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: volumeSize, - }, - }, - }, - }, - }, + VolumeClaimTemplates: volumeClaimTemplates, }, } diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index e8aa20103..e2bb3a70d 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -12,6 +12,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "time" "strconv" + "strings" +) + +const ( + MAX_DataNodePodRestartTime = 400 // in seconds + MAX_EsCommunicationTime = 180 // in seconds + MAX_EsWaitForDataNode = "6m" // time for the master to wait for data node to to reboot before it rebalances the shards + ) func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) error { @@ -21,10 +29,9 @@ func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) e //logrus.Infof("Scaling: statefulset : %v ", newstatefulset) ss_version , _ := strconv.Atoi(newstatefulset.ObjectMeta.ResourceVersion) - pod, _ := k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) - logrus.Infof("Scaling: NEW POD ",pod.ObjectMeta.ResourceVersion) - waitSeconds := 400 + logrus.Infof("Scaling: Pod revision ",pod.ObjectMeta.ResourceVersion) + waitSeconds := MAX_DataNodePodRestartTime for i := 0; i < waitSeconds; i++ { pod, _ = k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) if (pod == nil){ @@ -32,8 +39,22 @@ func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) e }else{ pod_version , _ := strconv.Atoi(pod.ObjectMeta.ResourceVersion) if pod_version > ss_version { - logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",pod.Status.ContainerStatuses) - if strings.Contains(dn.runningStatus,"ContainerStateRunning") { + //logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",string(pod.Status.ContainerStatuses)) + + status := "" + if (len(pod.Status.ContainerStatuses) > 0) { + if (pod.Status.ContainerStatuses[0].State.Running != nil) { + status = status + pod.Status.ContainerStatuses[0].State.Running.String() + } + if (pod.Status.ContainerStatuses[0].State.Waiting != nil) { + status = status + pod.Status.ContainerStatuses[0].State.Waiting.String() + } + if (pod.Status.ContainerStatuses[0].State.Terminated != nil) { + status = status + pod.Status.ContainerStatuses[0].State.Terminated.String() + } + } + logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",pod.Status.Phase," Status:",status) + if strings.Contains(status,"ContainerStateRunning") { ret = nil break } @@ -80,18 +101,18 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys masterip := "worked9d5000000.lab.es-c1.eastus2.us.azure.k8s.walmart.com:30004" // TODO: need to remove later - // Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. - if !es_set_delaytimeout(masterip) { - err := fmt.Errorf("Scaling: step-2 Setting delaytimeout failed") +// Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. + if err := es_checkForGreen(masterip); err != nil { + err = fmt.Errorf("Scaling: ES cluster is not in green state") return err } - // Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. - if err := es_checkForGreen(masterip); err != nil { // TODO : check green not implemented - err = fmt.Errorf("Scaling: step-3 ES cluster is not in green state") + +// Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. + if err := es_set_delaytimeout(masterip, MAX_EsWaitForDataNode);err != nil { // TODO before overwriting save the orginal setting return err } - // Step-4: scale the Data node by updating the new resources in the stateful set +// Step-4: scale the Data node by updating the new resources in the stateful set _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Update(statefulSet) if err != nil { logrus.Error("ERROR: Scaling: Could not scale statefulSet: ", err) @@ -99,21 +120,20 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys } dnodename := statefulSetName + "-0" - // Step-5: check if the POD is restarted and in running state from k8 point of view. +// Step-5: check if the POD is restarted and in running state from k8 point of view. k8_check_DataNodeRestarted(namespace, statefulSetName, k) - // Step-6: check if the POD is up from the ES point of view. - if err = es_checkForNodeUp(masterip, dnodename, 180); err != nil { +// Step-6: check if the POD is up from the ES point of view. + if err = es_checkForNodeUp(masterip, dnodename, MAX_EsCommunicationTime); err != nil { return err } - // Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. - if err = es_checkForShards(masterip, dnodename, 180); err != nil { +// Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. + if err = es_checkForShards(masterip, dnodename, MAX_EsCommunicationTime); err != nil { return err } - // Step-8: Undo the timeout settings - if !es_unset_delaytimeout(masterip) { - err = fmt.Errorf("Setting delaytimeout failed") +// Step-8: Undo the timeout settings + if err := es_set_delaytimeout(masterip, "1m");err != nil { return err } } From 96ad2ca11bcede7ba08e399896df9c4c0dfd30cc Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Fri, 23 Nov 2018 17:48:43 +0530 Subject: [PATCH 06/18] code cleanup. --- README.md | 46 +------- docs/scaling.md | 118 +++++++++++++++++++ pkg/apis/elasticsearchoperator/v1/cluster.go | 3 + pkg/k8sutil/es_crud.go | 106 +++-------------- pkg/k8sutil/k8sutil.go | 6 +- pkg/k8sutil/scaling.go | 36 +++--- pkg/processor/processor.go | 23 ++-- 7 files changed, 172 insertions(+), 166 deletions(-) create mode 100644 docs/scaling.md diff --git a/README.md b/README.md index 5fd5e8f85..0c09b1892 100644 --- a/README.md +++ b/README.md @@ -284,52 +284,8 @@ $ go get -u github.com/cloudflare/cfssl/cmd/cfssljson $ go run cmd/operator/main.go --kubecfg-file=${HOME}/.kube/config ``` # Scaling feature: -The following are changes present in scaling patch - -- Every Data node will have stateful set: currently all data nodes are part of one Statefule set, scaling needs every data node to be seperate statefulset with one replica, every datanode resource are updated independently to corresponding statefulset, so that the control will be in the hands of operator instead of k8. -- Scaling is optional feature: Seperate section is defined for scaling as shown in below example spec. If the scaling section is not present then entire scaling feature will be disabled. -- when Scaling will be triggered: Scaling will be triggered if there is any change in the one of the following 3 fields inside the scaling section. javaoptions and resources entries corresponds only to non-data nodes incase scaling section is present. If scaling section is abscent then it corresponds to all nodes. - - JavaOptions: This is the new field present inside the scaling section corresponds only to Data nodes. - - CPU inside resources : number of cpu cores. - - Memory inside resources : Memory size. - - Steps involved in vertical scaling of Elastic cluster: Repeating the following steps for each data node one after another, if there is any failure rest of scaling will be halted. - - Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory. - - Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. - - Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. - - Step-4: scale the Data node by updating the new resources in the stateful set - - Step-5: check if the POD is restarted and in running state from k8 point of view. - - Step-6: check if the POD is up from the ES point of view - - Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. - - Step-8: Undo the timeout settings. - - Future Enhancements: - - Horizontal scaling of Data nodes: increasing the "data-node-replica" will function as expected, but if "data-node-replica" is decreased by more then 2 then the Elastic search cluster can enter in to red state and there will be data loss, this can prevented by executing similar to vertical scaling one after another. - - -``` -Example Spec containing optional scaling section - -spec: - client-node-replicas: 3 - data-node-replicas: 3 - data-volume-size: 10Gi - java-options: -Xms256m -Xmx256m - master-node-replicas: 2 - scaling: - java-options: -Xms1052m -Xmx1052m - resources: - limits: - cpu: 2m - memory: 2048Mi - requests: - cpu: 1m - memory: 1024Mi - zones: - - us-east-1a - - us-east-1b - - us-east-1c -``` - +[Details of scaling](docs/scaling.md) # About Built by UPMC Enterprises in Pittsburgh, PA. http://enterprises.upmc.com/ diff --git a/docs/scaling.md b/docs/scaling.md new file mode 100644 index 000000000..a859296eb --- /dev/null +++ b/docs/scaling.md @@ -0,0 +1,118 @@ + +# Patch Details: + +The following are changes present in scaling patch + +- Every Data node will have stateful set: currently all data nodes are part of one Statefule set, scaling needs every data node to be seperate statefulset with one replica, every datanode resource are updated independently to corresponding statefulset, so that the control will be in the hands of operator instead of k8. +- Scaling is optional feature: Seperate section is defined for scaling as shown in below example spec. If the scaling section is not present then entire scaling feature will be disabled. +- Added changes to support local disk, but the is tested for multiple nodes. +- when Scaling will be triggered: Scaling will be triggered if there is any change in the one of the following 3 fields inside the scaling section. javaoptions and resources entries corresponds only to non-data nodes incase scaling section is present. If scaling section is abscent then it corresponds to all nodes. + - JavaOptions: This is the new field present inside the scaling section corresponds only to Data nodes. + - CPU inside resources : number of cpu cores. + - Memory inside resources : Memory size. + - Steps involved in vertical scaling of Elastic cluster: Repeating the following steps for each data node one after another, if there is any failure rest of scaling will be halted. + - Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory. + - Step-2: ES-change: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. + - Step-3: ES-change: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. + - Step-4: scale the Data node by updating the new resources in the statefull set + - Step-5: check if the POD is restarted and in running state from k8 point of view. + - Step-6: check if the POD is up from the ES point of view + - Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. + - Step-8: Undo the timeout settings. + - Future Enhancements: + - Horizontal scaling of Data nodes: increasing the "data-node-replica" will function as expected, but if "data-node-replica" is decreased by more then 2 then the Elastic search cluster can enter in to red state and there will be data loss, this can prevented by executing similar to vertical scaling one after another without entering into red state. + - Picking "masternodeip" from services k8 objects instead from user. + - Vertical scaling improvements: periodic scaling: currently vertical scaling is triggered from user, instead it can be triggered based on time automatically. + - Multiple threads: Currently vertical for each elasticsearch cluster takes considerable amount of time, during this time other elastic cluster MAY not be done, this can be parallelised by running scaling operation in multiple threads. + - elasticsearch api's TODO: change is settings assuming the default settings. + - Depends on ELasticSearch Version: It it depends on elasticsearch version as 6.3, if there is lot of changes in api's then scaling operation will fail. + - Local disk support: added changes to support local disk , but not tested with multiple nodes, it need to check that the pod will have affinity to the node, this can be enforced during the stateful set creation.. + - Usecases: + - case1: scale down during start of non-peak time , and scale-up during the start of peak time every day. doing scaling operation twice in a day, this can done using a cron job without interrupting the service. + - case2: scale down and scale up once in a while. + + +``` +Example Spec containing optional scaling section + +spec: + client-node-replicas: 3 + data-node-replicas: 3 + data-volume-size: 10Gi + java-options: -Xms256m -Xmx256m + master-node-replicas: 2 + scaling: + java-options: -Xms1052m -Xmx1052m + masternodeip: "10.12.16.132:30002" + resources: + limits: + cpu: 2m + memory: 2048Mi + requests: + cpu: 1m + memory: 1024Mi + zones: + - us-east-1a + - us-east-1b + - us-east-1c +``` + +# Log : + +Below is actual log when there is a trigger for scaling. + +``` +INFO[0032] Found cluster: es-cluster +INFO[0032] use-ssl not specified, defaulting to UseSSL=true +INFO[0032] Using [hub.docker.prod.walmart.com/upmcenterprises/docker-elasticsearch-kubernetes:6.1.3_0] as image for es cluster +INFO[0032] use-ssl not specified, defaulting to UseSSL=true +INFO[0034] Current replicas: %!(EXTRA int32=1, string= New replica: , int32=1) +INFO[0035] updated Stateful set: %!(EXTRA string=es-master-es-cluster-localdisk-0) +INFO[0035] Scaling enabled: true JavaOptions:-Xms1072m -Xmx1072m Resources: {{1024Mi 4m} {2048Mi 6m}} masterIP: 10.12.17.168:30004 +INFO[0035] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-0) +INFO[0035] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-0 +INFO[0035] Scaling: ES checking for green +INFO[0035] Scaling: ES checking for shards name: +INFO[0036] Scaling: change ES setting: delay timeout:6m and translog durability: request +INFO[0042] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-0 +INFO[0047] Scaling: POD started pod_version: 107346%!(EXTRA string= ss_version: %d, int=107321, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:30:54 +0530 IST,}) +INFO[0047] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-0-0 joined master +INFO[0057] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-0-0 +INFO[0058] Scaling: change ES setting: delay timeout:1m and translog durability: async +INFO[0058] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-0 :-------------------------------- +INFO[0058] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-1) +INFO[0058] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-1 +INFO[0058] Scaling: ES checking for green +INFO[0058] Scaling: ES checking for shards name: +INFO[0059] Scaling: change ES setting: delay timeout:6m and translog durability: request +INFO[0060] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-1 +INFO[0069] Scaling: POD started pod_version: 107396%!(EXTRA string= ss_version: %d, int=107369, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:31:16 +0530 IST,}) +INFO[0069] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-1-0 joined master +INFO[0079] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-1-0 +INFO[0079] Scaling: change ES setting: delay timeout:1m and translog durability: async +INFO[0080] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-1 :-------------------------------- +INFO[0080] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-2) +INFO[0080] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-2 +INFO[0080] Scaling: ES checking for green +INFO[0080] Scaling: ES checking for shards name: +INFO[0080] Scaling: change ES setting: delay timeout:6m and translog durability: request +INFO[0082] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-2 +INFO[0089] Scaling: POD started pod_version: 107448%!(EXTRA string= ss_version: %d, int=107421, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:31:36 +0530 IST,}) +INFO[0089] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-2-0 joined master +INFO[0099] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-2-0 +INFO[0099] Scaling: change ES setting: delay timeout:1m and translog durability: async +INFO[0100] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-2 :-------------------------------- +INFO[0100] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-3) +INFO[0100] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-3 +INFO[0100] Scaling: ES checking for green +INFO[0100] Scaling: ES checking for shards name: +INFO[0100] Scaling: change ES setting: delay timeout:6m and translog durability: request +INFO[0102] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-3 +INFO[0108] Scaling: POD started pod_version: 107498%!(EXTRA string= ss_version: %d, int=107470, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:31:56 +0530 IST,}) +INFO[0108] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-3-0 joined master +INFO[0118] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-3-0 +INFO[0120] Scaling: change ES setting: delay timeout:1m and translog durability: async +INFO[0120] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-3 :-------------------------------- +INFO[0122] --------> ElasticSearch Event finished! + +``` diff --git a/pkg/apis/elasticsearchoperator/v1/cluster.go b/pkg/apis/elasticsearchoperator/v1/cluster.go index 4d2acc0a8..62fb3cd4c 100644 --- a/pkg/apis/elasticsearchoperator/v1/cluster.go +++ b/pkg/apis/elasticsearchoperator/v1/cluster.go @@ -144,6 +144,9 @@ type Scaling struct { // JavaOptions defines args passed to elastic nodes JavaOptions string `json:"java-options"` + + // master node IP:port TODO: later this can be removed and learn dynamically from the service meta data. + MasterNodeIP string `json:"masternodeip"` } // Snapshot defines all params to create / store snapshots diff --git a/pkg/k8sutil/es_crud.go b/pkg/k8sutil/es_crud.go index ca730dabc..8c76c638f 100644 --- a/pkg/k8sutil/es_crud.go +++ b/pkg/k8sutil/es_crud.go @@ -10,33 +10,29 @@ import ( "net/http" "strings" "time" - "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" ) -/* TODO-1 : This speeds up scaling. -"persistent" : { - "cluster.routing.allocation.node_concurrent_recoveries": 20, from 2 to 20 - } - -TODO-2: - set translog durabilty to request: index.translog.durability - -After scaling operation it can be converted to async. +/* +TODO: + Assuming orginal settings are default values like delayed_timeout is 1m and translog.durability is async, incase if the user is changed then + user changed settings will be lost after scaling operations. + this can be corrected by reading the setting before change. */ -func es_set_delaytimeout(es_ip , duration string)(error){ +func es_change_settings(es_ip , duration,translog_durability string)(error){ var ret error ret = errors.New("Scaling: error in setting delay timeout") - - logrus.Infof("Scaling: setting ES delay timeout to ", duration) + + logrus.Infof("Scaling: change ES setting: delay timeout:%s and translog durability: %s", duration, translog_durability) client := &http.Client{ } body:="{ " - body = body + "\"settings\": { \"index.unassigned.node_left.delayed_timeout\": \""+duration+"\" }" + body = body + "\"settings\": { \"index.unassigned.node_left.delayed_timeout\": \""+duration+"\" ," + body = body + "\"index.translog.durability\": \""+translog_durability +"\" }" body = body + " }" - //fmt.Println("set BODY :",body,":") req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_all/_settings", bytes.NewBufferString(body)) req.Header.Add("Content-Type", `application/json`) resp, _ := client.Do(req) @@ -74,7 +70,7 @@ func util_wordcount(input string, nodename string) (int,int) { initializing_count++ } } - fmt.Println(nodename," shards: ",node_count," UNASSIGNED shards: ",unassigned_count," initialising shards: ",initializing_count) + //fmt.Println(nodename," shards: ",node_count," UNASSIGNED shards: ",unassigned_count," initialising shards: ",initializing_count) return node_count,unassigned_count+initializing_count } @@ -82,7 +78,7 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { var ret error ret = errors.New("still unassigned shards are there") - logrus.Infof("Scaling: ES checking for shards") + logrus.Infof("Scaling: ES checking for shards name: %s",nodeName) for i := 0; i < waitSeconds; i++ { response, err := http.Get("http://" + es_ip + "/_cat/shards") if err != nil { @@ -108,18 +104,16 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { var ret error ret = errors.New("ES node is not up") - logrus.Infof("Scaling: ES checking if the data node joined master") + logrus.Infof("Scaling: ES checking if the data node: %s joined master ",nodeName) for i := 0; i < waitSeconds; i++ { response, err := http.Get("http://" + es_ip + "/_cat/nodes?v&h=n,ip,v") if err != nil { - fmt.Printf("The HTTP request failed with error %s\n", err) + fmt.Printf("Scaling: The HTTP request failed with error %s\n", err) ret = err } else { data, _ := ioutil.ReadAll(response.Body) str_ret := strings.Contains(string(data), nodeName) if (str_ret){ - //time.Sleep(1 * time.Second) - //fmt.Println("http output: nodename: ",nodeName," DATA: ", string(data)) ret = nil return ret; } @@ -129,75 +123,5 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { return ret; } -/* -es_flush: response: &{409 Conflict 409 HTTP/1.1 1 1 map[Content-Type:[application/json; charset=UTF-8]] 0xc420017a60 -1 [] false true map[] 0xc4200d9600 } BODY: {"_shards":{"total":32,"successful":31,"failed":1}, -"indexer_tenant_1":{"total":16,"successful":16,"failed":0},"indexer_tenant_0":{"total":16,"successful":15,"failed":1,"failures":[{"shard":4,"reason":"pending operations","routing":{"state":"STARTED","primary":false,"node":"pGFMewaYSG-acz3GHywEzQ","relocating_node":null,"shard":4,"index":"indexer_tenant_0","allocation_id":{"id":"fu9axLfdQZqbwBKXd4DSPw"}}}]}} - */ - -func es_flush(es_ip string)(bool){ - ret := false - - client := &http.Client{ - } - - req, _ := http.NewRequest("POST", "http://"+es_ip+"/_all/_flush/synced", bytes.NewBufferString("")) - req.Header.Add("Content-Type", `application/json`) - resp, _ := client.Do(req) - data, _ := ioutil.ReadAll(resp.Body) - fmt.Println("es_flush: response: ",resp," BODY: ",string(data)) - if (resp.StatusCode == 200){ - ret = true - time.Sleep(1 * time.Second) - } - return ret; -} - -func es_set_readonly(es_ip string)(bool){ - ret := false -// return true - - client := &http.Client{ - } - body:="{ " - //body= body +"{ \"persistent\": { \"cluster.blocks.read_only\": \"true\" } }" - body = body + "\"transient\": { \"cluster.blocks.read_only\": \"true\" }" -// body = body + ", \"cluster.routing.allocation.enable\": \"none\" } " - body = body + " }" - - //fmt.Println("set BODY :",body,":") - req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_cluster/settings", bytes.NewBufferString(body)) - req.Header.Add("Content-Type", `application/json`) - resp, _ := client.Do(req) - fmt.Println("es_flush: response: ",resp, resp.Body) - if (resp.StatusCode == 200){ - ret = true - time.Sleep(1 * time.Second) - } - es_flush(es_ip) - - return ret; -} -func es_unset_readonly(es_ip string)(bool){ - ret := false -// return true - client := &http.Client{ - } - body:="{ " - //body= body +"{ \"persistent\": { \"cluster.blocks.read_only\": \"false\" } }" - body = body + "\"transient\": { \"cluster.blocks.read_only\": \"false\" }" - //body = body + ", \"cluster.routing.allocation.enable\": \"all\" } " - body = body + " }" - fmt.Println("unset BODY :",body,":") - req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_cluster/settings", bytes.NewBufferString(body)) - req.Header.Add("Content-Type", `application/json`) - resp, _ := client.Do(req) - fmt.Println("es_unset: response: ",resp) - if (resp.StatusCode == 200){ - ret = true - time.Sleep(1 * time.Second) - } - - return ret; -} \ No newline at end of file diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 90813c9e3..3a34608b7 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -670,7 +670,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s statefulSet.Spec.Template.Spec.ServiceAccountName = serviceAccountName } - if storageClass != "default" { + if storageClass != "default" && storageClass != "localdisk"{ statefulSet.Spec.VolumeClaimTemplates[0].Annotations = map[string]string{ "volume.beta.kubernetes.io/storage-class": storageClass, } @@ -681,7 +681,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s // CreateDataNodeDeployment creates the data node deployment func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources, - imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string, index int,scaling bool) error { + imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string, index int,scaling bool,scalingMasterIP string) error { deploymentName, _, _, _ := processDeploymentType(deploymentType, clusterName) @@ -707,7 +707,7 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3 return err } if deploymentType == "data" && scaling { - return scale_datanode(k, namespace, statefulSetName, resources, javaOptions, statefulSet) + return scale_datanode(k, namespace, statefulSetName, resources, javaOptions, statefulSet,scalingMasterIP) } //scale replicas? if statefulSet.Spec.Replicas != replicas { diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index e2bb3a70d..6084206ec 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -18,24 +18,22 @@ import ( const ( MAX_DataNodePodRestartTime = 400 // in seconds MAX_EsCommunicationTime = 180 // in seconds - MAX_EsWaitForDataNode = "6m" // time for the master to wait for data node to to reboot before it rebalances the shards - + MAX_EsWaitForDataNode = "6m" // time for the master to wait for data node to reboot before it rebalances the shards ) func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) error { ret := fmt.Errorf("Scaling: POD is not up: ",statefulSetName) dnodename := statefulSetName + "-0" newstatefulset, _ := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) - //logrus.Infof("Scaling: statefulset : %v ", newstatefulset) ss_version , _ := strconv.Atoi(newstatefulset.ObjectMeta.ResourceVersion) pod, _ := k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) - logrus.Infof("Scaling: Pod revision ",pod.ObjectMeta.ResourceVersion) + logrus.Infof("Scaling: checking Datanode if it restarted or not by k8: %s",statefulSetName) waitSeconds := MAX_DataNodePodRestartTime for i := 0; i < waitSeconds; i++ { pod, _ = k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) if (pod == nil){ - logrus.Infof("Scaling: POD Terminated ") + //logrus.Infof("Scaling: POD Terminated ") }else{ pod_version , _ := strconv.Atoi(pod.ObjectMeta.ResourceVersion) if pod_version > ss_version { @@ -53,28 +51,27 @@ func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) e status = status + pod.Status.ContainerStatuses[0].State.Terminated.String() } } - logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",pod.Status.Phase," Status:",status) if strings.Contains(status,"ContainerStateRunning") { + logrus.Infof("Scaling: POD started pod_version: %d",pod_version," ss_version: %d",ss_version," Status: %s",status) + ret = nil break } - }else{ - logrus.Infof("Scaling: pod revision: ", pod.ObjectMeta.ResourceVersion, " ss revision: ", newstatefulset.ObjectMeta.ResourceVersion) } } - time.Sleep(5 * time.Second) + time.Sleep(1 * time.Second) } return ret } -func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet) error { +func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet, masterip string) error { cpu, _ := resource.ParseQuantity(resources.Requests.CPU) memory, _ := resource.ParseQuantity(resources.Requests.Memory) // Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory resourceMatch := true if cpu != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] { - logrus.Infof("Scaling: step-1 : CPU Changed USER: ", cpu, " from k8: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) + logrus.Infof("Scaling: cpu changed by user: ", cpu, " current value: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) resourceMatch = false statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] = cpu } @@ -86,7 +83,7 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys for index, env := range statefulSet.Spec.Template.Spec.Containers[0].Env { if env.Name == "ES_JAVA_OPTS" { if env.Value != javaOptions { - logrus.Infof("Scaling: step-1 JAVA OPTS Changed USER: ", javaOptions, " from k8: ", env.Value) + logrus.Infof("Scaling: java_opts Changed by user: ", javaOptions, " current value: ", env.Value, " name: ",statefulSetName) resourceMatch = false statefulSet.Spec.Template.Spec.Containers[0].Env[index].Value = javaOptions } @@ -94,21 +91,19 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys } } - if !resourceMatch { - logrus.Infof("Scaling: step-1 RESOURCES CHANGED: updating Stateful set: ", statefulSetName) + if !resourceMatch { + logrus.Infof("Scaling:STARTED scaling with new resources ... : %s", statefulSetName) // TODO : only memory request is updated, memory limit as need to be updated. - statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memory - - masterip := "worked9d5000000.lab.es-c1.eastus2.us.azure.k8s.walmart.com:30004" // TODO: need to remove later + statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memory // Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. if err := es_checkForGreen(masterip); err != nil { - err = fmt.Errorf("Scaling: ES cluster is not in green state") + err = fmt.Errorf("Scaling: ES cluster is not in green state", " name: ",statefulSetName) return err } // Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. - if err := es_set_delaytimeout(masterip, MAX_EsWaitForDataNode);err != nil { // TODO before overwriting save the orginal setting + if err := es_change_settings(masterip, MAX_EsWaitForDataNode, "request");err != nil { // TODO before overwriting save the orginal setting return err } @@ -133,9 +128,10 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys return err } // Step-8: Undo the timeout settings - if err := es_set_delaytimeout(masterip, "1m");err != nil { + if err := es_change_settings(masterip, "1m", "async");err != nil { return err } + logrus.Infof("Scaling:------------- sucessfully Completed for: %s :--------------------------------", statefulSetName) } return nil } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 0acc64f12..bc5db62d6 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -173,6 +173,7 @@ func (p *Processor) refreshClusters() error { }, Scaling: myspec.Scaling{ JavaOptions: cluster.Spec.Scaling.JavaOptions, + MasterNodeIP: cluster.Spec.Scaling.MasterNodeIP, Resources: myspec.Resources{ Limits: myspec.MemoryCPU{ Memory: cluster.Spec.Scaling.Resources.Limits.Memory, @@ -344,7 +345,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Is a base image defined in the custom cluster? var baseImage = p.calcBaseImage(p.baseImage, c.Spec.ElasticSearchImage) - logrus.Infof("JANA Using [%s] as image for es cluster", baseImage) + logrus.Infof(" Using [%s] as image for es cluster", baseImage) // Default UseSSL to true useSSL := p.defaultUseSSL(c.Spec.UseSSL) @@ -403,7 +404,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionMaster { if err := p.k8sclient.CreateDataNodeDeployment("master", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,0 , false); err != nil { + c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,0 , false,""); err != nil { logrus.Error("Error creating master node deployment ", err) return err } @@ -421,11 +422,15 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) resources = c.Spec.Scaling.Resources scaling = true } - logrus.Info("Scaling enabled: ",scaling," zone distribution JavaOptions:",javaOptions," Resources: ",resources) + scalingMasterIP := "" + if scaling { + scalingMasterIP = c.Spec.Scaling.MasterNodeIP + } + logrus.Info("Scaling enabled: ",scaling," zone distribution JavaOptions:",javaOptions," Resources: ",resources, "masterip: ",scalingMasterIP) for index, count := range zoneDistributionData { if err := p.k8sclient.CreateDataNodeDeployment("data", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,index,scaling); err != nil { + c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,index,scaling,scalingMasterIP); err != nil { logrus.Error("Error creating data node deployment ", err) return err @@ -441,7 +446,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Create Master Nodes if err := p.k8sclient.CreateDataNodeDeployment("master", func() *int32 { i := int32(c.Spec.MasterNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, 0, false); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, 0, false,""); err != nil { logrus.Error("Error creating master node deployment ", err) return err @@ -460,11 +465,15 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) resources = c.Spec.Scaling.Resources scaling = true } - logrus.Info("Scaling enabled: ",scaling," JavaOptions:",javaOptions," Resources: ",resources," ESURL:",c.Spec.Scheduler.ElasticURL) + scalingMasterIP := "" + if scaling { + scalingMasterIP = c.Spec.Scaling.MasterNodeIP + } + logrus.Info("Scaling enabled: ",scaling," JavaOptions:",javaOptions," Resources: ",resources," masterIP: ",scalingMasterIP) for nodeindex := 0; nodeindex < dataNodeCount; nodeindex++ { if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(1); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, nodeindex , scaling); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, nodeindex , scaling, scalingMasterIP); err != nil { logrus.Error("Error creating data node deployment ", err) return err } From 388b21f9ef2ddda45b02011d4a991b3b75e0ddf8 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Fri, 23 Nov 2018 19:07:41 +0530 Subject: [PATCH 07/18] compilation errors addressed. --- pkg/k8sutil/scaling.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index 6084206ec..2b3e25c1b 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -22,7 +22,7 @@ const ( ) func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) error { - ret := fmt.Errorf("Scaling: POD is not up: ",statefulSetName) + ret := fmt.Errorf("Scaling: POD is not up: ") dnodename := statefulSetName + "-0" newstatefulset, _ := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) @@ -98,7 +98,7 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys // Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. if err := es_checkForGreen(masterip); err != nil { - err = fmt.Errorf("Scaling: ES cluster is not in green state", " name: ",statefulSetName) + err = fmt.Errorf("Scaling: ES cluster is not in green state") return err } From 67218cc25e224846f558e569952ef2b0129d045b Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Fri, 23 Nov 2018 20:47:03 +0530 Subject: [PATCH 08/18] Document changes. --- docs/scaling.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/scaling.md b/docs/scaling.md index a859296eb..0b7ba1b8a 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -5,28 +5,29 @@ The following are changes present in scaling patch - Every Data node will have stateful set: currently all data nodes are part of one Statefule set, scaling needs every data node to be seperate statefulset with one replica, every datanode resource are updated independently to corresponding statefulset, so that the control will be in the hands of operator instead of k8. - Scaling is optional feature: Seperate section is defined for scaling as shown in below example spec. If the scaling section is not present then entire scaling feature will be disabled. -- Added changes to support local disk, but the is tested for multiple nodes. -- when Scaling will be triggered: Scaling will be triggered if there is any change in the one of the following 3 fields inside the scaling section. javaoptions and resources entries corresponds only to non-data nodes incase scaling section is present. If scaling section is abscent then it corresponds to all nodes. - - JavaOptions: This is the new field present inside the scaling section corresponds only to Data nodes. +- Added changes to support local disk, but it is not tested for multiple nodes, MAY need to add affinity between POD and node. +- when Scaling will be triggered: Scaling will be triggered if there is any change in the one of the following 3 fields inside the scaling section: javaoptions,cpu and memory. javaoptions and resources entries corresponds only to non-data nodes incase scaling section is present. If scaling section is abscent then it corresponds to all nodes. + - JavaOptions: - CPU inside resources : number of cpu cores. - Memory inside resources : Memory size. - - Steps involved in vertical scaling of Elastic cluster: Repeating the following steps for each data node one after another, if there is any failure rest of scaling will be halted. - - Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory. - - Step-2: ES-change: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. + - Steps involved in vertical scaling of Elastic cluster: Repeating the following steps for each data node one after another, if there is any failure then scaling will be halted for the rest of data nodes: + - Step-1: check if there is any changes in 3-resources inside scaling section: javaoptions,cpu and memory. + - Step-2: ES-setting-change: change default time from 1 min to 6 min to avoid copying of shards belonging to the data node that is going to be scaled. + - step-2: ES-setting-change: change translong durability from async to sync/request basis, this is to make no data is loss. - Step-3: ES-change: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. - - Step-4: scale the Data node by updating the new resources in the statefull set + - Step-4: scale the Data node by updating the new resources in the statefull set. Here the Data node will be restarted. - Step-5: check if the POD is restarted and in running state from k8 point of view. - - Step-6: check if the POD is up from the ES point of view + - Step-6: check if the POD is up from the ES point of view. means Data node is register with the master. - Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. - - Step-8: Undo the timeout settings. + - Step-8: Undo the settings done in step-2. - Future Enhancements: - Horizontal scaling of Data nodes: increasing the "data-node-replica" will function as expected, but if "data-node-replica" is decreased by more then 2 then the Elastic search cluster can enter in to red state and there will be data loss, this can prevented by executing similar to vertical scaling one after another without entering into red state. - Picking "masternodeip" from services k8 objects instead from user. - Vertical scaling improvements: periodic scaling: currently vertical scaling is triggered from user, instead it can be triggered based on time automatically. - Multiple threads: Currently vertical for each elasticsearch cluster takes considerable amount of time, during this time other elastic cluster MAY not be done, this can be parallelised by running scaling operation in multiple threads. - elasticsearch api's TODO: change is settings assuming the default settings. - - Depends on ELasticSearch Version: It it depends on elasticsearch version as 6.3, if there is lot of changes in api's then scaling operation will fail. - - Local disk support: added changes to support local disk , but not tested with multiple nodes, it need to check that the pod will have affinity to the node, this can be enforced during the stateful set creation.. + - Dependency on ElasticSearch Version: It it depends on elasticsearch version as 6.3, if there is lot of changes in api's then scaling operation will fail. + - Local disk support: added changes to support local disk, but not tested with multiple nodes, it need to check that the pod will have affinity to the node, this can be enforced during the statefull set creation.. - Usecases: - case1: scale down during start of non-peak time , and scale-up during the start of peak time every day. doing scaling operation twice in a day, this can done using a cron job without interrupting the service. - case2: scale down and scale up once in a while. @@ -59,7 +60,7 @@ spec: # Log : -Below is actual log when there is a trigger for scaling. +Below is actual log when there is a trigger for scaling, This is for a four data node cluster. ``` INFO[0032] Found cluster: es-cluster From 06bed8678cd1a0dcb4490376af7425335f5063b0 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Sat, 24 Nov 2018 14:15:05 +0530 Subject: [PATCH 09/18] code cleanup. --- docs/scaling.md | 2 +- pkg/k8sutil/deployments.go | 12 ++++++------ pkg/k8sutil/es_crud.go | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/scaling.md b/docs/scaling.md index 0b7ba1b8a..7b3feab6c 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -25,7 +25,7 @@ The following are changes present in scaling patch - Picking "masternodeip" from services k8 objects instead from user. - Vertical scaling improvements: periodic scaling: currently vertical scaling is triggered from user, instead it can be triggered based on time automatically. - Multiple threads: Currently vertical for each elasticsearch cluster takes considerable amount of time, during this time other elastic cluster MAY not be done, this can be parallelised by running scaling operation in multiple threads. - - elasticsearch api's TODO: change is settings assuming the default settings. + - elasticsearch api's TODO: reverse in settings assuming the default settings. - Dependency on ElasticSearch Version: It it depends on elasticsearch version as 6.3, if there is lot of changes in api's then scaling operation will fail. - Local disk support: added changes to support local disk, but not tested with multiple nodes, it need to check that the pod will have affinity to the node, this can be enforced during the statefull set creation.. - Usecases: diff --git a/pkg/k8sutil/deployments.go b/pkg/k8sutil/deployments.go index b6e6755e1..205bb7a31 100644 --- a/pkg/k8sutil/deployments.go +++ b/pkg/k8sutil/deployments.go @@ -122,11 +122,11 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java limitMemory, _ := resource.ParseQuantity(resources.Limits.Memory) requestCPU, _ := resource.ParseQuantity(resources.Requests.CPU) requestMemory, _ := resource.ParseQuantity(resources.Requests.Memory) -// scheme := v1.URISchemeHTTP + scheme := v1.URISchemeHTTP if useSSL != nil && *useSSL { -// scheme = v1.URISchemeHTTPS + scheme = v1.URISchemeHTTPS } - /*probe := &v1.Probe{ + probe := &v1.Probe{ TimeoutSeconds: 30, InitialDelaySeconds: 10, FailureThreshold: 15, @@ -137,7 +137,7 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java Scheme: scheme, }, }, - }*/ + } deployment := &v1beta1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, @@ -239,8 +239,8 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java Protocol: v1.ProtocolTCP, }, }, - //ReadinessProbe: probe, - //LivenessProbe: probe, + ReadinessProbe: probe, + LivenessProbe: probe, VolumeMounts: []v1.VolumeMount{ v1.VolumeMount{ Name: "storage", diff --git a/pkg/k8sutil/es_crud.go b/pkg/k8sutil/es_crud.go index 8c76c638f..90973278b 100644 --- a/pkg/k8sutil/es_crud.go +++ b/pkg/k8sutil/es_crud.go @@ -51,9 +51,9 @@ func es_change_settings(es_ip , duration,translog_durability string)(error){ } return ret; } -func es_checkForGreen(es_ip string)(error){ /* TODO */ +func es_checkForGreen(es_ip string)(error){ logrus.Infof("Scaling: ES checking for green ") - return es_checkForShards(es_ip , "", 180) + return es_checkForShards(es_ip , "", MAX_EsCommunicationTime) } func util_wordcount(input string, nodename string) (int,int) { node_count := 0 From 890e5113021754da4d550ce5280381cd602a88d3 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Sat, 24 Nov 2018 18:19:34 +0530 Subject: [PATCH 10/18] main.go changes. --- cmd/operator/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 5ca5b63cd..b2f4cd809 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -35,9 +35,9 @@ import ( "syscall" "github.com/Sirupsen/logrus" -// "github.com/heptiolabs/healthcheck" -// "github.com/prometheus/client_golang/prometheus" -// "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/heptiolabs/healthcheck" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/upmc-enterprises/elasticsearch-operator/pkg/controller" "github.com/upmc-enterprises/elasticsearch-operator/pkg/k8sutil" @@ -102,17 +102,17 @@ func Main() int { doneChan := make(chan struct{}) var wg sync.WaitGroup -/* JANA + r := prometheus.NewRegistry() r.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "")) r.MustRegister(prometheus.NewGoCollector()) health := healthcheck.NewMetricsHandler(r, "elasticsearch-operator") -*/ + mux := http.NewServeMux() -// mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{})) -// mux.HandleFunc("/live", health.LiveEndpoint) -// mux.HandleFunc("/ready", health.ReadyEndpoint) + mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{})) + mux.HandleFunc("/live", health.LiveEndpoint) + mux.HandleFunc("/ready", health.ReadyEndpoint) // Kick it off controller.Run() From 086522143e7de77af895ec454af4e9c7443e281c Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Mon, 26 Nov 2018 22:37:29 +0530 Subject: [PATCH 11/18] learning masterip from service dynamically. --- docs/scaling.md | 2 -- pkg/apis/elasticsearchoperator/v1/cluster.go | 3 --- pkg/k8sutil/k8sutil.go | 4 +-- pkg/k8sutil/scaling.go | 28 ++++++++++++++++++-- pkg/processor/processor.go | 23 ++++++---------- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/docs/scaling.md b/docs/scaling.md index 7b3feab6c..4705cbea3 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -25,7 +25,6 @@ The following are changes present in scaling patch - Picking "masternodeip" from services k8 objects instead from user. - Vertical scaling improvements: periodic scaling: currently vertical scaling is triggered from user, instead it can be triggered based on time automatically. - Multiple threads: Currently vertical for each elasticsearch cluster takes considerable amount of time, during this time other elastic cluster MAY not be done, this can be parallelised by running scaling operation in multiple threads. - - elasticsearch api's TODO: reverse in settings assuming the default settings. - Dependency on ElasticSearch Version: It it depends on elasticsearch version as 6.3, if there is lot of changes in api's then scaling operation will fail. - Local disk support: added changes to support local disk, but not tested with multiple nodes, it need to check that the pod will have affinity to the node, this can be enforced during the statefull set creation.. - Usecases: @@ -44,7 +43,6 @@ spec: master-node-replicas: 2 scaling: java-options: -Xms1052m -Xmx1052m - masternodeip: "10.12.16.132:30002" resources: limits: cpu: 2m diff --git a/pkg/apis/elasticsearchoperator/v1/cluster.go b/pkg/apis/elasticsearchoperator/v1/cluster.go index 62fb3cd4c..4d2acc0a8 100644 --- a/pkg/apis/elasticsearchoperator/v1/cluster.go +++ b/pkg/apis/elasticsearchoperator/v1/cluster.go @@ -144,9 +144,6 @@ type Scaling struct { // JavaOptions defines args passed to elastic nodes JavaOptions string `json:"java-options"` - - // master node IP:port TODO: later this can be removed and learn dynamically from the service meta data. - MasterNodeIP string `json:"masternodeip"` } // Snapshot defines all params to create / store snapshots diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 3a34608b7..0dc30b541 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -681,7 +681,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s // CreateDataNodeDeployment creates the data node deployment func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources, - imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string, index int,scaling bool,scalingMasterIP string) error { + imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string, index int,scaling bool) error { deploymentName, _, _, _ := processDeploymentType(deploymentType, clusterName) @@ -707,7 +707,7 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3 return err } if deploymentType == "data" && scaling { - return scale_datanode(k, namespace, statefulSetName, resources, javaOptions, statefulSet,scalingMasterIP) + return scale_datanode(k, namespace, clusterName, statefulSetName, resources, javaOptions, statefulSet) } //scale replicas? if statefulSet.Spec.Replicas != replicas { diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index 2b3e25c1b..5dd048906 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -63,8 +63,26 @@ func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) e } return ret } - -func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet, masterip string) error { +func get_masterIP(k *K8sutil, namespace, clusterName string)(string){ + listN, _ := k.Kclient.CoreV1().Services(namespace).List(metav1.ListOptions{}) + for _, s := range listN.Items { + service_name := "elasticsearch-"+ clusterName + if (service_name == s.Name){ + if (len(s.Spec.ExternalIPs)> 0 && (len(s.Spec.Ports) > 0)){ + ret := s.Spec.ExternalIPs[0] + ":"+ strconv.Itoa(int(s.Spec.Ports[0].NodePort)) + logrus.Infof("Scaling: MasterIP external port: %s",ret) + return ret + } + if (len(s.Spec.ClusterIP)> 0 && (len(s.Spec.Ports) > 0)){ + ret := s.Spec.ClusterIP+ ":"+ strconv.Itoa(int(s.Spec.Ports[0].Port)) + logrus.Infof("Scaling: MasterIP Internal port: %s",ret) + return ret + } + } + } + return "" +} +func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet ) error { cpu, _ := resource.ParseQuantity(resources.Requests.CPU) memory, _ := resource.ParseQuantity(resources.Requests.Memory) @@ -91,6 +109,12 @@ func scale_datanode(k *K8sutil, namespace, statefulSetName string, resources mys } } + masterip := get_masterIP(k,namespace,clusterName) + if (masterip == ""){ + err := fmt.Errorf("MasterIP is empty") + return err + } + if !resourceMatch { logrus.Infof("Scaling:STARTED scaling with new resources ... : %s", statefulSetName) // TODO : only memory request is updated, memory limit as need to be updated. diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index bc5db62d6..154134a78 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -173,7 +173,6 @@ func (p *Processor) refreshClusters() error { }, Scaling: myspec.Scaling{ JavaOptions: cluster.Spec.Scaling.JavaOptions, - MasterNodeIP: cluster.Spec.Scaling.MasterNodeIP, Resources: myspec.Resources{ Limits: myspec.MemoryCPU{ Memory: cluster.Spec.Scaling.Resources.Limits.Memory, @@ -404,7 +403,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionMaster { if err := p.k8sclient.CreateDataNodeDeployment("master", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,0 , false,""); err != nil { + c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,0 , false); err != nil { logrus.Error("Error creating master node deployment ", err) return err } @@ -422,15 +421,12 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) resources = c.Spec.Scaling.Resources scaling = true } - scalingMasterIP := "" - if scaling { - scalingMasterIP = c.Spec.Scaling.MasterNodeIP - } - logrus.Info("Scaling enabled: ",scaling," zone distribution JavaOptions:",javaOptions," Resources: ",resources, "masterip: ",scalingMasterIP) + + logrus.Info("Scaling enabled: ",scaling," zone distribution JavaOptions:",javaOptions," Resources: ",resources) for index, count := range zoneDistributionData { if err := p.k8sclient.CreateDataNodeDeployment("data", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,index,scaling,scalingMasterIP); err != nil { + c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,index,scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err @@ -446,7 +442,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Create Master Nodes if err := p.k8sclient.CreateDataNodeDeployment("master", func() *int32 { i := int32(c.Spec.MasterNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, 0, false,""); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, 0, false); err != nil { logrus.Error("Error creating master node deployment ", err) return err @@ -465,15 +461,12 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) resources = c.Spec.Scaling.Resources scaling = true } - scalingMasterIP := "" - if scaling { - scalingMasterIP = c.Spec.Scaling.MasterNodeIP - } - logrus.Info("Scaling enabled: ",scaling," JavaOptions:",javaOptions," Resources: ",resources," masterIP: ",scalingMasterIP) + + logrus.Info("Scaling enabled: ",scaling," JavaOptions:",javaOptions," Resources: ",resources) for nodeindex := 0; nodeindex < dataNodeCount; nodeindex++ { if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(1); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, nodeindex , scaling, scalingMasterIP); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, nodeindex , scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err } From 4cd85d806fb677609e81fbf3b71b1226b61543c1 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Fri, 30 Nov 2018 17:33:22 +0530 Subject: [PATCH 12/18] All data nodes in a single statefulset. --- pkg/k8sutil/k8sutil.go | 14 ++- pkg/k8sutil/scaling.go | 219 +++++++++++++++++++++++++------------ pkg/processor/processor.go | 23 ++-- 3 files changed, 165 insertions(+), 91 deletions(-) diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 0dc30b541..84b755be5 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -446,7 +446,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s volumeClaimTemplates := []v1.PersistentVolumeClaim{} if storageClass == "localdisk" { hostpath := "/mnt/"+statefulSetName - // hosttype := "Directory" + // TODO: localdisk does not work, since the local path is same for all data nodes, this need to be fixed. volumes = []v1.Volume{ v1.Volume{ Name: "es-data", @@ -496,6 +496,9 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s }, Spec: apps.StatefulSetSpec{ Replicas: replicas, + UpdateStrategy: apps.StatefulSetUpdateStrategy { + Type: apps.OnDeleteStatefulSetStrategyType, + }, ServiceName: statefulSetName, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -681,11 +684,11 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s // CreateDataNodeDeployment creates the data node deployment func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources, - imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string, index int,scaling bool) error { + imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string,scaling bool) error { deploymentName, _, _, _ := processDeploymentType(deploymentType, clusterName) - statefulSetName := fmt.Sprintf("%s-%s-%d", deploymentName, storageClass, index) + statefulSetName := fmt.Sprintf("%s-%s", deploymentName, storageClass) // Check if StatefulSet exists statefulSet, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) @@ -707,12 +710,12 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3 return err } if deploymentType == "data" && scaling { - return scale_datanode(k, namespace, clusterName, statefulSetName, resources, javaOptions, statefulSet) + ret := scale_statefulset(k, namespace, clusterName, statefulSetName, resources, javaOptions, statefulSet, *replicas) + return ret } //scale replicas? if statefulSet.Spec.Replicas != replicas { currentReplicas := *statefulSet.Spec.Replicas - logrus.Infof(" Current replicas: ",currentReplicas," New replica: ",*replicas) if *replicas < currentReplicas { minMasterNodes := elasticsearchutil.MinMasterNodes(int(*replicas)) logrus.Infof("Detected master scale-down. Setting 'discovery.zen.minimum_master_nodes' to %d", minMasterNodes) @@ -720,7 +723,6 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3 } statefulSet.Spec.Replicas = replicas _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Update(statefulSet) - logrus.Infof(" updated Stateful set: ", statefulSetName) if err != nil { logrus.Error("Could not scale statefulSet: ", err) minMasterNodes := elasticsearchutil.MinMasterNodes(int(currentReplicas)) diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index 5dd048906..54c8f8088 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -3,6 +3,7 @@ */ package k8sutil + import ( "fmt" "github.com/Sirupsen/logrus" @@ -10,52 +11,91 @@ import ( "k8s.io/api/apps/v1beta2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "time" + //"k8s.io/api/core/v1" "strconv" "strings" + "time" ) const ( - MAX_DataNodePodRestartTime = 400 // in seconds - MAX_EsCommunicationTime = 180 // in seconds - MAX_EsWaitForDataNode = "6m" // time for the master to wait for data node to reboot before it rebalances the shards + MAX_DataNodePodRestartTime = 400 // in seconds + MAX_EsCommunicationTime = 180 // in seconds + MAX_EsWaitForDataNode = "6m" // time for the master to wait for data node to reboot before it rebalances the shards ) +func compare_pod_vs_statefulset(namespace, statefulSetName,podName string, k *K8sutil) bool { + resourceMatch := true + logrus.Infof("Scaling: Compare Spec POD Vs statefulset : ",podName) + statefulSet, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) + if (err != nil){ + return resourceMatch + } + pod, err := k.Kclient.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{}) + if (err != nil){ + return resourceMatch + } + + if pod.Spec.Containers[0].Resources.Requests["cpu"] != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] { + logrus.Infof("Scaling: Compare POD Vs SS cpu changed by user: ", pod.Spec.Containers[0].Resources.Requests["cpu"], " current value: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) + resourceMatch = false + return resourceMatch + } + /*if memory != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] { + resourceMatch = false + return resourceMatch + }*/ + javaOptions:="" + for _, env := range pod.Spec.Containers[0].Env { + if env.Name == "ES_JAVA_OPTS" { + javaOptions = env.Value + break + } + } + for _, env := range statefulSet.Spec.Template.Spec.Containers[0].Env { + if env.Name == "ES_JAVA_OPTS" { + if env.Value != javaOptions { + logrus.Infof("Scaling: Compare POD Vs SS java_opts Changed by user: ", javaOptions, " current value: ", env.Value, " name: ", podName) + resourceMatch = false + return resourceMatch + } + break + } + } + + return resourceMatch +} -func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) error { +func k8_check_DataNodeRestarted(namespace, statefulSetName,dnodename string, k *K8sutil) error { ret := fmt.Errorf("Scaling: POD is not up: ") - dnodename := statefulSetName + "-0" - newstatefulset, _ := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) - - ss_version , _ := strconv.Atoi(newstatefulset.ObjectMeta.ResourceVersion) + statefulset, _ := k.Kclient.AppsV1beta2().StatefulSets(namespace).Get(statefulSetName, metav1.GetOptions{}) + + ss_version, _ := strconv.Atoi(statefulset.ObjectMeta.ResourceVersion) pod, _ := k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) - logrus.Infof("Scaling: checking Datanode if it restarted or not by k8: %s",statefulSetName) + logrus.Infof("Scaling: checking Datanode if it restarted or not by k8:ss: %s datanode:%s", statefulSetName,dnodename) waitSeconds := MAX_DataNodePodRestartTime for i := 0; i < waitSeconds; i++ { pod, _ = k.Kclient.CoreV1().Pods(namespace).Get(dnodename, metav1.GetOptions{}) - if (pod == nil){ + if pod == nil { //logrus.Infof("Scaling: POD Terminated ") - }else{ - pod_version , _ := strconv.Atoi(pod.ObjectMeta.ResourceVersion) + } else { + pod_version, _ := strconv.Atoi(pod.ObjectMeta.ResourceVersion) if pod_version > ss_version { - //logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",string(pod.Status.ContainerStatuses)) - + //logrus.Infof("Scaling: POD started pod_version:",pod_version," dd_version:",ss_version," state: ",(pod.Status.ContainerStatuses)) status := "" - if (len(pod.Status.ContainerStatuses) > 0) { - if (pod.Status.ContainerStatuses[0].State.Running != nil) { + if len(pod.Status.ContainerStatuses) > 0 { + if pod.Status.ContainerStatuses[0].State.Running != nil { status = status + pod.Status.ContainerStatuses[0].State.Running.String() } - if (pod.Status.ContainerStatuses[0].State.Waiting != nil) { + if pod.Status.ContainerStatuses[0].State.Waiting != nil { status = status + pod.Status.ContainerStatuses[0].State.Waiting.String() } - if (pod.Status.ContainerStatuses[0].State.Terminated != nil) { + if pod.Status.ContainerStatuses[0].State.Terminated != nil { status = status + pod.Status.ContainerStatuses[0].State.Terminated.String() } } - if strings.Contains(status,"ContainerStateRunning") { - logrus.Infof("Scaling: POD started pod_version: %d",pod_version," ss_version: %d",ss_version," Status: %s",status) - + if strings.Contains(status, "ContainerStateRunning") { + logrus.Infof("Scaling: POD started pod_version: %d", pod_version, " ss_version: %d", ss_version, " Status: %s", status) ret = nil - break + break } } } @@ -63,30 +103,74 @@ func k8_check_DataNodeRestarted(namespace, statefulSetName string, k *K8sutil) e } return ret } -func get_masterIP(k *K8sutil, namespace, clusterName string)(string){ +func get_masterIP(k *K8sutil, namespace, clusterName string) string { listN, _ := k.Kclient.CoreV1().Services(namespace).List(metav1.ListOptions{}) for _, s := range listN.Items { - service_name := "elasticsearch-"+ clusterName - if (service_name == s.Name){ - if (len(s.Spec.ExternalIPs)> 0 && (len(s.Spec.Ports) > 0)){ - ret := s.Spec.ExternalIPs[0] + ":"+ strconv.Itoa(int(s.Spec.Ports[0].NodePort)) - logrus.Infof("Scaling: MasterIP external port: %s",ret) + service_name := "elasticsearch-" + clusterName + if service_name == s.Name { + if len(s.Spec.ExternalIPs) > 0 && (len(s.Spec.Ports) > 0) { + ret := s.Spec.ExternalIPs[0] + ":" + strconv.Itoa(int(s.Spec.Ports[0].NodePort)) + logrus.Infof("Scaling: MasterIP external port: %s", ret) return ret } - if (len(s.Spec.ClusterIP)> 0 && (len(s.Spec.Ports) > 0)){ - ret := s.Spec.ClusterIP+ ":"+ strconv.Itoa(int(s.Spec.Ports[0].Port)) - logrus.Infof("Scaling: MasterIP Internal port: %s",ret) + if len(s.Spec.ClusterIP) > 0 && (len(s.Spec.Ports) > 0) { + ret := s.Spec.ClusterIP + ":" + strconv.Itoa(int(s.Spec.Ports[0].Port)) + logrus.Infof("Scaling: MasterIP Internal port: %s", ret) return ret } } } return "" } -func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet ) error { +func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName, podName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet, pod_index int32, masterip string) error { + // Step-3: ES-change: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. + if err := es_checkForGreen(masterip); err != nil { + err = fmt.Errorf("Scaling: ES cluster is not in green state") + return err + } + + // Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. + if err := es_change_settings(masterip, MAX_EsWaitForDataNode, "request"); err != nil { // TODO before overwriting save the orginal setting + return err + } + + // Step-4: scale the Data node by deleting the POD + deletePolicy := metav1.DeletePropagationForeground + + err := k.Kclient.Core().Pods(namespace).Delete(podName, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy}) + if err != nil { + logrus.Error("ERROR: Scaling: Could not delete POD: ", err) + return err + } + + // Step-5: check if the POD is restarted and in running state from k8 point of view. + if err = k8_check_DataNodeRestarted(namespace, statefulSetName, podName, k); err != nil { + return err + } + + // Step-6: check if the POD is up from the ES point of view. + if err = es_checkForNodeUp(masterip, podName, MAX_EsCommunicationTime); err != nil { + return err + } + + // Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. + if err = es_checkForShards(masterip, podName, MAX_EsCommunicationTime); err != nil { + return err + } + // Step-8: Undo the timeout settings + if err := es_change_settings(masterip, "1m", "async"); err != nil { + return err + } + logrus.Infof("Scaling:------------- sucessfully Completed for: %s :--------------------------------", podName) + return nil +} + +func scale_statefulset(k *K8sutil, namespace, clusterName, statefulSetName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet, replicas int32) error { cpu, _ := resource.ParseQuantity(resources.Requests.CPU) memory, _ := resource.ParseQuantity(resources.Requests.Memory) -// Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory + // Step-1: check if there is any change in the 3-resources: javaoptions,cpu and memory resourceMatch := true if cpu != statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"] { logrus.Infof("Scaling: cpu changed by user: ", cpu, " current value: ", statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["cpu"]) @@ -101,7 +185,7 @@ func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName string, for index, env := range statefulSet.Spec.Template.Spec.Containers[0].Env { if env.Name == "ES_JAVA_OPTS" { if env.Value != javaOptions { - logrus.Infof("Scaling: java_opts Changed by user: ", javaOptions, " current value: ", env.Value, " name: ",statefulSetName) + logrus.Infof("Scaling: java_opts Changed by user: ", javaOptions, " current value: ", env.Value, " name: ", statefulSetName) resourceMatch = false statefulSet.Spec.Template.Spec.Containers[0].Env[index].Value = javaOptions } @@ -109,53 +193,44 @@ func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName string, } } - masterip := get_masterIP(k,namespace,clusterName) - if (masterip == ""){ - err := fmt.Errorf("MasterIP is empty") - return err + masterip := get_masterIP(k, namespace, clusterName) + if masterip == "" { + err := fmt.Errorf("MasterIP is empty") + return err } - - if !resourceMatch { + + if !resourceMatch { logrus.Infof("Scaling:STARTED scaling with new resources ... : %s", statefulSetName) // TODO : only memory request is updated, memory limit as need to be updated. - statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memory - -// Step-3: ES-chanage: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. - if err := es_checkForGreen(masterip); err != nil { - err = fmt.Errorf("Scaling: ES cluster is not in green state") - return err - } + statefulSet.Spec.Template.Spec.Containers[0].Resources.Requests["memory"] = memory -// Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. - if err := es_change_settings(masterip, MAX_EsWaitForDataNode, "request");err != nil { // TODO before overwriting save the orginal setting - return err - } - -// Step-4: scale the Data node by updating the new resources in the stateful set _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Update(statefulSet) if err != nil { - logrus.Error("ERROR: Scaling: Could not scale statefulSet: ", err) - return err - } - dnodename := statefulSetName + "-0" - -// Step-5: check if the POD is restarted and in running state from k8 point of view. - k8_check_DataNodeRestarted(namespace, statefulSetName, k) - -// Step-6: check if the POD is up from the ES point of view. - if err = es_checkForNodeUp(masterip, dnodename, MAX_EsCommunicationTime); err != nil { + logrus.Error("ERROR: Scaling: Could not update statefulSet: ", err) return err } - -// Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. - if err = es_checkForShards(masterip, dnodename, MAX_EsCommunicationTime); err != nil { - return err + + var index int32 = 0 + for index = 0; index < replicas; index++ { + podName := fmt.Sprintf("%s-%d", statefulSetName, index) + ret := scale_datanode(k, namespace, clusterName, statefulSetName, podName, resources, javaOptions, statefulSet, index,masterip) + if ret != nil { + return ret + } } -// Step-8: Undo the timeout settings - if err := es_change_settings(masterip, "1m", "async");err != nil { - return err + }else{ + // check if Scaling is stopped half the way, this may be because server is stopped or for various reasons */ + var index int32 = 0 + for index = 0; index < replicas; index++ { + podName := fmt.Sprintf("%s-%d", statefulSetName, index) + if (compare_pod_vs_statefulset(namespace, statefulSetName, podName, k)){ + continue + } + ret := scale_datanode(k, namespace, clusterName, statefulSetName, podName, resources, javaOptions, statefulSet, index,masterip) + if ret != nil { + return ret + } } - logrus.Infof("Scaling:------------- sucessfully Completed for: %s :--------------------------------", statefulSetName) } return nil } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 154134a78..8158ae2ae 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -27,7 +27,6 @@ package processor import ( "fmt" "sync" - //"runtime/debug" "github.com/upmc-enterprises/elasticsearch-operator/pkg/elasticsearchutil" "github.com/upmc-enterprises/elasticsearch-operator/pkg/snapshot" @@ -333,8 +332,8 @@ func (p *Processor) processMasterPodEvent(c *v1.Pod) error { } func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) error { - logrus.Println("--------> Received ElasticSearch Event!:", c) - //debug.PrintStack() + logrus.Println("--------> Received ElasticSearch Event!") + // Refresh if err := p.refreshClusters(); err != nil { logrus.Error("Error refreshing cluster ", err) @@ -344,7 +343,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Is a base image defined in the custom cluster? var baseImage = p.calcBaseImage(p.baseImage, c.Spec.ElasticSearchImage) - logrus.Infof(" Using [%s] as image for es cluster", baseImage) + logrus.Infof("Using [%s] as image for es cluster", baseImage) // Default UseSSL to true useSSL := p.defaultUseSSL(c.Spec.UseSSL) @@ -403,7 +402,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionMaster { if err := p.k8sclient.CreateDataNodeDeployment("master", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,0 , false); err != nil { + c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, false); err != nil { logrus.Error("Error creating master node deployment ", err) return err } @@ -426,7 +425,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionData { if err := p.k8sclient.CreateDataNodeDeployment("data", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,index,scaling); err != nil { + c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err @@ -442,14 +441,13 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Create Master Nodes if err := p.k8sclient.CreateDataNodeDeployment("master", func() *int32 { i := int32(c.Spec.MasterNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, 0, false); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, false); err != nil { logrus.Error("Error creating master node deployment ", err) return err } // Create Data Nodes - dataNodeCount := int(c.Spec.DataNodeReplicas) /* TODO: if the replica count is decreased, then need to handle accordingly */ javaOptions := c.Spec.JavaOptions resources := c.Spec.Resources scaling := false @@ -463,14 +461,13 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) } logrus.Info("Scaling enabled: ",scaling," JavaOptions:",javaOptions," Resources: ",resources) - for nodeindex := 0; nodeindex < dataNodeCount; nodeindex++ { - if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(1); return &i }(), baseImage, c.Spec.Storage.StorageClass, + + if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, nodeindex , scaling); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL , scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err - } - } + } } // Deploy Kibana From 791dc6f99621b99f8b0f306be40a5d84e0526285 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Thu, 20 Dec 2018 15:48:19 +0530 Subject: [PATCH 13/18] Test Cases added to doc/scaling.md --- docs/scaling.md | 65 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/docs/scaling.md b/docs/scaling.md index 4705cbea3..c6f19d520 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -3,7 +3,6 @@ The following are changes present in scaling patch -- Every Data node will have stateful set: currently all data nodes are part of one Statefule set, scaling needs every data node to be seperate statefulset with one replica, every datanode resource are updated independently to corresponding statefulset, so that the control will be in the hands of operator instead of k8. - Scaling is optional feature: Seperate section is defined for scaling as shown in below example spec. If the scaling section is not present then entire scaling feature will be disabled. - Added changes to support local disk, but it is not tested for multiple nodes, MAY need to add affinity between POD and node. - when Scaling will be triggered: Scaling will be triggered if there is any change in the one of the following 3 fields inside the scaling section: javaoptions,cpu and memory. javaoptions and resources entries corresponds only to non-data nodes incase scaling section is present. If scaling section is abscent then it corresponds to all nodes. @@ -35,27 +34,69 @@ The following are changes present in scaling patch ``` Example Spec containing optional scaling section +apiVersion: enterprises.upmc.com/v1 +kind: ElasticsearchCluster +metadata: + clusterName: "" + creationTimestamp: 2018-11-19T08:49:23Z + generation: 1 + name: es-cluster + namespace: default + resourceVersion: "1636643" + selfLink: /apis/enterprises.upmc.com/v1/namespaces/default/elasticsearchclusters/es-cluster + uid: 03267c65-ebd8-11e8-8e6a-000d3a000cf8 spec: - client-node-replicas: 3 - data-node-replicas: 3 + cerebro: + image: hub.docker.prod.walmart.com/upmcenterprises/cerebro:0.6.8 + client-node-replicas: 1 + data-node-replicas: 4 data-volume-size: 10Gi - java-options: -Xms256m -Xmx256m - master-node-replicas: 2 + elastic-search-image: quay.docker.prod.walmart.com/pires/docker-elasticsearch-kubernetes:6.3.2 + java-options: -Xms1052m -Xmx1052m + master-node-replicas: 1 + network-host: 0.0.0.0 + resources: + limits: + cpu: 4m + memory: 3048Mi + requests: + cpu: 3m + memory: 2024Mi scaling: - java-options: -Xms1052m -Xmx1052m + java-options: -Xms1078m -Xmx1078m resources: limits: - cpu: 2m + cpu: 5m memory: 2048Mi requests: - cpu: 1m + cpu: 4m memory: 1024Mi - zones: - - us-east-1a - - us-east-1b - - us-east-1c + storage: + storage-class: standard-disk + zones: [] ``` +# TestCases : + +- Testcase-1 : Normal scaling operation on Network block storage. + - Description: Scaling operation can be triggered by changing the scaling parameters in ElasticSearch Cluster configuration,this can be done using "kubectl edit ..". If there is any change in jvm-heap memory in java-options or cpu cores inside the scaling section then scaling operation will be triggered. This can be observed in elastic search operator log. During scaling operation, the following should be full filled: a) At any time only one data node should be restarted, b) The State of ElasticSearch cluster should not be entered into Red state anytime during scaling operation. In case if the Elasticsearch cluster enter in to Redstate then the scaling operation will be automatically halted by the elasaticsearch opeartor. + - Test Status: Passed. +- Testcase-2 : Stop and start the ES operator during scaling operation. + - Description: when the scaling operation is halfway, stop the elasticsearch operator. example: out 20 nodes, 10 nodes have completed scaling, during that time stop the elastic operator, and start the elasticsearch operator after few minutes, when the elasticsearch operator is started then scaling of the rest of nodes should continue where it was stopped last time instead of starting from the beginning. + - Test Status: Passed. +- Testcase-3 : Trigger scaling operation when Elastic cluster is in "Yellow/Red" state + - Description: When the Elastic cluster in non-green state, and if scaling operation is started it should not start scaling operation. means not single data pod should be restarted. Scaling operation should be started only if the Elastic cluster is in green state. + - Test Status : Passed. +- Testcase-4 : Trigger of scaling operation: changes in Non-scaling parameter + - Description: If there is any change in non-scalar parameter, then scaling operation should not be triggered. Scaling operation should be triggered only of there is change in Java-options, cpu or memory inside scaling section. To trigger scaling atleast any one of jvm heap or cpu is required. + - Test Status: Passed +- Testcase-5 : Normal scaling operation on local/nfs storage. + - Description: Similar to Testcase-1 except localdisk is set as storage-class instead of network block storage. + - Test status : Failed . + - Reason for Failure: Since all data nodes share same statefull set, the mount point for all data nodes is same, due to this second data nodes will not comesup, this need to addressed in future. + + + # Log : Below is actual log when there is a trigger for scaling, This is for a four data node cluster. From ed52b799998563f860258d7d95e4f7a59bb4ec5a Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Sat, 22 Dec 2018 13:55:37 +0530 Subject: [PATCH 14/18] Detail steps for each testcase are added. --- docs/scaling.md | 119 ++++++++++++++++++------------------------------ 1 file changed, 44 insertions(+), 75 deletions(-) diff --git a/docs/scaling.md b/docs/scaling.md index c6f19d520..704aa220f 100644 --- a/docs/scaling.md +++ b/docs/scaling.md @@ -78,81 +78,50 @@ spec: # TestCases : -- Testcase-1 : Normal scaling operation on Network block storage. - - Description: Scaling operation can be triggered by changing the scaling parameters in ElasticSearch Cluster configuration,this can be done using "kubectl edit ..". If there is any change in jvm-heap memory in java-options or cpu cores inside the scaling section then scaling operation will be triggered. This can be observed in elastic search operator log. During scaling operation, the following should be full filled: a) At any time only one data node should be restarted, b) The State of ElasticSearch cluster should not be entered into Red state anytime during scaling operation. In case if the Elasticsearch cluster enter in to Redstate then the scaling operation will be automatically halted by the elasaticsearch opeartor. - - Test Status: Passed. -- Testcase-2 : Stop and start the ES operator during scaling operation. - - Description: when the scaling operation is halfway, stop the elasticsearch operator. example: out 20 nodes, 10 nodes have completed scaling, during that time stop the elastic operator, and start the elasticsearch operator after few minutes, when the elasticsearch operator is started then scaling of the rest of nodes should continue where it was stopped last time instead of starting from the beginning. - - Test Status: Passed. -- Testcase-3 : Trigger scaling operation when Elastic cluster is in "Yellow/Red" state - - Description: When the Elastic cluster in non-green state, and if scaling operation is started it should not start scaling operation. means not single data pod should be restarted. Scaling operation should be started only if the Elastic cluster is in green state. - - Test Status : Passed. -- Testcase-4 : Trigger of scaling operation: changes in Non-scaling parameter - - Description: If there is any change in non-scalar parameter, then scaling operation should not be triggered. Scaling operation should be triggered only of there is change in Java-options, cpu or memory inside scaling section. To trigger scaling atleast any one of jvm heap or cpu is required. - - Test Status: Passed -- Testcase-5 : Normal scaling operation on local/nfs storage. - - Description: Similar to Testcase-1 except localdisk is set as storage-class instead of network block storage. - - Test status : Failed . - - Reason for Failure: Since all data nodes share same statefull set, the mount point for all data nodes is same, due to this second data nodes will not comesup, this need to addressed in future. - - - -# Log : +# Testcase-1 : Normal scaling operation on Network block storage. + - Description: Scaling operation can be triggered by changing the scaling parameters in ElasticSearch Cluster configuration,this can be done using "kubectl edit ..". If there is any change in jvm-heap memory in java-options or cpu cores inside the scaling section then scaling operation will be triggered. This can be observed in elastic search operator log. During scaling operation, the following should be full filled: a) At any time only one data node should be restarted, b) The State of ElasticSearch cluster should not be entered into Red state anytime during scaling operation. In case if the Elasticsearch cluster enter in to Redstate then the scaling operation will be automatically halted by the elasaticsearch opeartor. +- Steps to Reproduce the test: + - step-1: edit the configuration of elastic cluster, this can done using "kubectl edit,..", change the parameters like jvm heap memory size or cpu cores inside the scaling section and save the file. + - step-2: After step-1, elastic search operator will receive the signal about the change in configuration of cluster, then the scaling code will check if there is any change in parameters related to scaling, this triggers scaling operation, this can be monitored in operator log. + - step-3: elasticsearch operator will update the data node pods one after another, and making sure the elastic search cluster is in yellow/green. +- Test Status: Passed. -Below is actual log when there is a trigger for scaling, This is for a four data node cluster. +# Testcase-2 : Stop and start the ES operator during scaling operation. +- Description: when the scaling operation is halfway, stop the elasticsearch operator. example: out 20 nodes, 10 nodes have completed scaling, during that time stop the elastic operator, and start the elasticsearch operator after few minutes, when the elasticsearch operator is started then scaling of the rest of nodes should continue where it was stopped last time instead of starting from the beginning. +- Steps to Reproduce the test: + - step-1: edit the configuration of elastic cluster, this can done using "kubectl edit,..", change the parameters like jvm heap memory size or cpu cores inside the scaling section and save the file. + - step-2: After step-1, elastic search operator will receive the signal about the change in configuration of cluster, then the scaling code will check if there is any change in parameters related to scaling, this triggers scaling operation, this can be monitored in operator log. + - step-3: Elasticsearch operator will update the data node pods one after another, and making sure the elastic search cluster is in yellow/green. + - step-4: After scaling is half way of the data nodes, stop the elastic search operator.example: out of 20 nodes, after completing 10 nodes, stop the ES operator. + - step-5: Wait for 10 min, and start ES operator + - step-6: When the ES operator is started, rest of the datanodes will get scaled without starting from the start. + - step-7: check in elasticsearch cluster the starting time of each data node. half of the data nodes would have started 10 min later. + - Test Status: Passed. -``` -INFO[0032] Found cluster: es-cluster -INFO[0032] use-ssl not specified, defaulting to UseSSL=true -INFO[0032] Using [hub.docker.prod.walmart.com/upmcenterprises/docker-elasticsearch-kubernetes:6.1.3_0] as image for es cluster -INFO[0032] use-ssl not specified, defaulting to UseSSL=true -INFO[0034] Current replicas: %!(EXTRA int32=1, string= New replica: , int32=1) -INFO[0035] updated Stateful set: %!(EXTRA string=es-master-es-cluster-localdisk-0) -INFO[0035] Scaling enabled: true JavaOptions:-Xms1072m -Xmx1072m Resources: {{1024Mi 4m} {2048Mi 6m}} masterIP: 10.12.17.168:30004 -INFO[0035] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-0) -INFO[0035] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-0 -INFO[0035] Scaling: ES checking for green -INFO[0035] Scaling: ES checking for shards name: -INFO[0036] Scaling: change ES setting: delay timeout:6m and translog durability: request -INFO[0042] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-0 -INFO[0047] Scaling: POD started pod_version: 107346%!(EXTRA string= ss_version: %d, int=107321, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:30:54 +0530 IST,}) -INFO[0047] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-0-0 joined master -INFO[0057] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-0-0 -INFO[0058] Scaling: change ES setting: delay timeout:1m and translog durability: async -INFO[0058] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-0 :-------------------------------- -INFO[0058] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-1) -INFO[0058] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-1 -INFO[0058] Scaling: ES checking for green -INFO[0058] Scaling: ES checking for shards name: -INFO[0059] Scaling: change ES setting: delay timeout:6m and translog durability: request -INFO[0060] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-1 -INFO[0069] Scaling: POD started pod_version: 107396%!(EXTRA string= ss_version: %d, int=107369, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:31:16 +0530 IST,}) -INFO[0069] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-1-0 joined master -INFO[0079] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-1-0 -INFO[0079] Scaling: change ES setting: delay timeout:1m and translog durability: async -INFO[0080] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-1 :-------------------------------- -INFO[0080] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-2) -INFO[0080] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-2 -INFO[0080] Scaling: ES checking for green -INFO[0080] Scaling: ES checking for shards name: -INFO[0080] Scaling: change ES setting: delay timeout:6m and translog durability: request -INFO[0082] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-2 -INFO[0089] Scaling: POD started pod_version: 107448%!(EXTRA string= ss_version: %d, int=107421, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:31:36 +0530 IST,}) -INFO[0089] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-2-0 joined master -INFO[0099] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-2-0 -INFO[0099] Scaling: change ES setting: delay timeout:1m and translog durability: async -INFO[0100] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-2 :-------------------------------- -INFO[0100] Scaling: java_opts Changed by user: %!(EXTRA string=-Xms1072m -Xmx1072m, string= current value: , string=-Xms1062m -Xmx1062m, string= name: , string=es-data-es-cluster-localdisk-3) -INFO[0100] Scaling:STARTED scaling with new resources ... : es-data-es-cluster-localdisk-3 -INFO[0100] Scaling: ES checking for green -INFO[0100] Scaling: ES checking for shards name: -INFO[0100] Scaling: change ES setting: delay timeout:6m and translog durability: request -INFO[0102] Scaling: checking Datanode if it restarted or not by k8: es-data-es-cluster-localdisk-3 -INFO[0108] Scaling: POD started pod_version: 107498%!(EXTRA string= ss_version: %d, int=107470, string= Status: %s, string=&ContainerStateRunning{StartedAt:2018-11-23 17:31:56 +0530 IST,}) -INFO[0108] Scaling: ES checking if the data node: es-data-es-cluster-localdisk-3-0 joined master -INFO[0118] Scaling: ES checking for shards name: es-data-es-cluster-localdisk-3-0 -INFO[0120] Scaling: change ES setting: delay timeout:1m and translog durability: async -INFO[0120] Scaling:------------- sucessfully Completed for: es-data-es-cluster-localdisk-3 :-------------------------------- -INFO[0122] --------> ElasticSearch Event finished! +# Testcase-3 : Trigger scaling operation when Elastic cluster is in "Yellow/Red" state +- Description: When the Elastic cluster in non-green state, and if scaling operation is started it should not start scaling operation. means not single data pod should be restarted. Scaling operation should be started only if the Elastic cluster is in green state. +- Steps to Reproduce the test: + - step-1: make the elastic search cluster to change in to yellow state by stopping one of the data node or by other means. + - step-2: edit the configuration of elastic cluster, this can done using "kubectl edit,..", change the parameters like jvm heap memory size or cpu cores inside the scaling section and save the file. + - step-3: After step-2, elastic search operator will receive the signal about the change in configuration of cluster, then the scaling code will check if there is any change in parameters related to scaling, this triggers scaling operation, this can be monitored in operator log. + - step-4: scaling of Data nodes will started, but immedietly it will generate an error saying ES cluster is in yellow state and scaling operation is aborted. +- Test Status : Passed. -``` +# Testcase-4 : Trigger of scaling operation: changes in Non-scaling parameter +- Description: If there is any change in non-scalar parameter, then scaling operation should not be triggered. Scaling operation should be triggered only of there is change in Java-options, cpu or memory inside scaling section. To trigger scaling atleast any one of jvm heap or cpu is required. +- Steps to Reproduce the test: + - step-1: edit the configuration of elastic cluster, this can done using "kubectl edit,..", change the parameters not related to scaling. + - step-2: After step-1, elastic search operator will receive the signal about the change in configuration of cluster, but scaling operaton of data nodes will not started saying there is no change in memory and cpu. +- Test Status: Passed + +# Testcase-5 : Normal scaling operation on local/nfs storage. +- Description: Similar to Testcase-1 except localdisk is set as storage-class instead of network block storage. +- Steps to Reproduce the test: + - step-1: edit the configuration of elastic cluster, this can done using "kubectl edit,..", change the parameters like jvm heap memory size or cpu cores inside the scaling section and save the file. + - step-2: After step-1, elastic search operator will receive the signal about the change in configuration of cluster, then the scaling code will check if there is any change in parameters related to scaling, this triggers scaling operation, this can be monitored in operator log. + - step-3: elasticsearch operator will update the data node pods one after another, and making sure the elastic search cluster is in yellow/green. +- Test status : Failed . +- Reason for Failure: Since all data nodes share same statefull set, the mount point for all data nodes is same, due to this second data nodes will not comesup, this need to addressed in future. + + + \ No newline at end of file From b8c9ee4d4a86391eebb1db73552476f1a5b3985b Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Thu, 3 Jan 2019 21:02:13 +0530 Subject: [PATCH 15/18] Unit test code and some fixes added. --- pkg/k8sutil/es_crud.go | 22 +++++++++++++++------- pkg/k8sutil/scaling.go | 10 +++++----- pkg/k8sutil/scaling_test.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 pkg/k8sutil/scaling_test.go diff --git a/pkg/k8sutil/es_crud.go b/pkg/k8sutil/es_crud.go index 90973278b..821f0001f 100644 --- a/pkg/k8sutil/es_crud.go +++ b/pkg/k8sutil/es_crud.go @@ -21,7 +21,7 @@ TODO: this can be corrected by reading the setting before change. */ -func es_change_settings(es_ip , duration,translog_durability string)(error){ +func ES_change_settings(es_ip , duration,translog_durability string)(error){ var ret error ret = errors.New("Scaling: error in setting delay timeout") @@ -36,6 +36,9 @@ func es_change_settings(es_ip , duration,translog_durability string)(error){ req, _ := http.NewRequest("PUT", "http://"+es_ip+"/_all/_settings", bytes.NewBufferString(body)) req.Header.Add("Content-Type", `application/json`) resp, _ := client.Do(req) + if (resp == nil){ + return ret; + } data, _ := ioutil.ReadAll(resp.Body) if (resp.StatusCode == 200){ @@ -51,9 +54,9 @@ func es_change_settings(es_ip , duration,translog_durability string)(error){ } return ret; } -func es_checkForGreen(es_ip string)(error){ +func ES_checkForGreen(es_ip string)(error){ logrus.Infof("Scaling: ES checking for green ") - return es_checkForShards(es_ip , "", MAX_EsCommunicationTime) + return ES_checkForShards(es_ip , "", MAX_EsCommunicationTime) } func util_wordcount(input string, nodename string) (int,int) { node_count := 0 @@ -74,7 +77,7 @@ func util_wordcount(input string, nodename string) (int,int) { return node_count,unassigned_count+initializing_count } -func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { +func ES_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { var ret error ret = errors.New("still unassigned shards are there") @@ -82,7 +85,7 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { for i := 0; i < waitSeconds; i++ { response, err := http.Get("http://" + es_ip + "/_cat/shards") if err != nil { - fmt.Printf("The HTTP request failed with error %s\n", err) + logrus.Infof("Error: The HTTP request failed with error %s\n", err) } else { data, _ := ioutil.ReadAll(response.Body) _,unassigned_count := util_wordcount(string(data), nodeName) @@ -90,7 +93,12 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { time.Sleep(1 * time.Second) continue } else { - ret = nil + + if (response.StatusCode == 200){ + ret = nil + }else{ + logrus.Infof("Scaling: Error checkForShards response :%s: statuscode %d:",data,response.StatusCode) + } break } } @@ -100,7 +108,7 @@ func es_checkForShards(es_ip string, nodeName string, waitSeconds int)(error) { return ret; } - func es_checkForNodeUp(es_ip string, nodeName string, waitSeconds int)(error){ + func ES_checkForNodeUp(es_ip string, nodeName string, waitSeconds int)(error){ var ret error ret = errors.New("ES node is not up") diff --git a/pkg/k8sutil/scaling.go b/pkg/k8sutil/scaling.go index 54c8f8088..72742bb61 100644 --- a/pkg/k8sutil/scaling.go +++ b/pkg/k8sutil/scaling.go @@ -124,13 +124,13 @@ func get_masterIP(k *K8sutil, namespace, clusterName string) string { } func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName, podName string, resources myspec.Resources, javaOptions string, statefulSet *v1beta2.StatefulSet, pod_index int32, masterip string) error { // Step-3: ES-change: check if ES cluster is green state, suppose if one of the data node is down and state is yellow then do not proceed with scaling. - if err := es_checkForGreen(masterip); err != nil { + if err := ES_checkForGreen(masterip); err != nil { err = fmt.Errorf("Scaling: ES cluster is not in green state") return err } // Step-2: ES-chanage: change default time from 1 min to 3 to n min to avoid copying of shards belonging to the data node that is going to be scaled. - if err := es_change_settings(masterip, MAX_EsWaitForDataNode, "request"); err != nil { // TODO before overwriting save the orginal setting + if err := ES_change_settings(masterip, MAX_EsWaitForDataNode, "request"); err != nil { // TODO before overwriting save the orginal setting return err } @@ -150,16 +150,16 @@ func scale_datanode(k *K8sutil, namespace, clusterName, statefulSetName, podName } // Step-6: check if the POD is up from the ES point of view. - if err = es_checkForNodeUp(masterip, podName, MAX_EsCommunicationTime); err != nil { + if err = ES_checkForNodeUp(masterip, podName, MAX_EsCommunicationTime); err != nil { return err } // Step-7: check if all shards are registered with Master, At this ES should turn in to green from yellow, now it is safe to scale next data node. - if err = es_checkForShards(masterip, podName, MAX_EsCommunicationTime); err != nil { + if err = ES_checkForShards(masterip, podName, MAX_EsCommunicationTime); err != nil { return err } // Step-8: Undo the timeout settings - if err := es_change_settings(masterip, "1m", "async"); err != nil { + if err := ES_change_settings(masterip, "1m", "async"); err != nil { return err } logrus.Infof("Scaling:------------- sucessfully Completed for: %s :--------------------------------", podName) diff --git a/pkg/k8sutil/scaling_test.go b/pkg/k8sutil/scaling_test.go new file mode 100644 index 000000000..0c4a59d68 --- /dev/null +++ b/pkg/k8sutil/scaling_test.go @@ -0,0 +1,37 @@ +package k8sutil + +import ( + "testing" + "github.com/upmc-enterprises/elasticsearch-operator/pkg/k8sutil" +) + + +const ( + NonExistingIP = "10.1.1.1" /* some non existing IP */ + NonExistingPOD = "nonexistingpodname" +) + +/* + Test for Failures, by sending the unrechable ES Master IP, the below function under test should always return the error, + incase if testing function does not return the error then the unit test should fail. +*/ +func Test_scaling_change_setting(t *testing.T) { + err := k8sutil.ES_change_settings(NonExistingIP, "1m", "request"); + if (err == nil){ + t.Errorf("Scaling unittest change setting failed"); + } +} + +func Test_check_for_green(t *testing.T) { + err := k8sutil.ES_checkForGreen(NonExistingIP); + if (err == nil){ + t.Errorf("Scaling unittest check_for_green failed"); + } +} + +func Test_check_for_nodeUp(t *testing.T) { + err := k8sutil.ES_checkForNodeUp(NonExistingIP, NonExistingPOD, 5); + if (err == nil){ + t.Errorf("Scaling unittest check_for_nodeUp failed"); + } +} From 730dbe3faa852cf18b80286ccc97854fe0b8b131 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Sun, 13 Jan 2019 16:34:44 +0530 Subject: [PATCH 16/18] resolving conflicts from the master. --- pkg/apis/elasticsearchoperator/v1/cluster.go | 18 +++++++++++++++++- pkg/k8sutil/k8sutil.go | 6 +++++- pkg/processor/processor.go | 11 ++++++----- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/apis/elasticsearchoperator/v1/cluster.go b/pkg/apis/elasticsearchoperator/v1/cluster.go index 4d2acc0a8..efa84be35 100644 --- a/pkg/apis/elasticsearchoperator/v1/cluster.go +++ b/pkg/apis/elasticsearchoperator/v1/cluster.go @@ -25,6 +25,7 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT package v1 import ( + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -72,6 +73,12 @@ type ClusterSpec struct { // labels. NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // Tolerations specifies which tolerations the Master and Data nodes will have applied to them + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + + // Affinity (podAffinity, podAntiAffinity, nodeAffinity) will be applied to the Client nodes + Affinity v1.Affinity `json:"affinity,omitempty"` + // Zones specifies a map of key-value pairs. Defines which zones // to deploy persistent volumes for data nodes Zones []string `json:"zones,omitempty"` @@ -95,8 +102,17 @@ type ClusterSpec struct { // Storage defines how volumes are provisioned Storage Storage `json:"storage"` - // JavaOptions defines args passed to elastic nodes + // JavaOptions defines args passed to all elastic nodes JavaOptions string `json:"java-options"` + + // ClientJavaOptions defines args passed to client nodes (Overrides JavaOptions) + ClientJavaOptions string `json:"client-java-options"` + + // DataJavaOptions defines args passed to data nodes (Overrides JavaOptions) + DataJavaOptions string `json:"data-java-options"` + + // MasterJavaOptions defines args passed to master nodes (Overrides JavaOptions) + MasterJavaOptions string `json:"master-java-options"` // ImagePullSecrets defines credentials to pull image from private repository (optional) ImagePullSecrets []ImagePullSecrets `json:"image-pull-secrets"` diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 84b755be5..fb6dbb6ba 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -584,6 +584,10 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s Name: "HTTP_ENABLE", Value: "true", }, + v1.EnvVar{ + Name: "path.data", + Value: "/data11/test", + }, v1.EnvVar{ Name: "SEARCHGUARD_SSL_TRANSPORT_ENABLED", Value: enableSSL, @@ -684,7 +688,7 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s // CreateDataNodeDeployment creates the data node deployment func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources, - imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string,scaling bool) error { + imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions, masterJavaOptions, dataJavaOptions string, useSSL *bool, esUrl string, nodeSelector map[string]string, tolerations []v1.Toleration, scaling bool) error { deploymentName, _, _, _ := processDeploymentType(deploymentType, clusterName) diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 8158ae2ae..bc9e7dd55 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -124,7 +124,8 @@ func (p *Processor) defaultUseSSL(specUseSSL *bool) bool { // Default to true if specUseSSL == nil { logrus.Infof("use-ssl not specified, defaulting to UseSSL=true") - return true + return false + // return true } else { logrus.Infof("use-ssl %v", *specUseSSL) return *specUseSSL @@ -402,7 +403,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionMaster { if err := p.k8sclient.CreateDataNodeDeployment("master", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, false); err != nil { + c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.MasterJavaOptions, c.Spec.DataJavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, c.Spec.NodeSelector, c.Spec.Tolerations, false); err != nil { logrus.Error("Error creating master node deployment ", err) return err } @@ -425,7 +426,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) for index, count := range zoneDistributionData { if err := p.k8sclient.CreateDataNodeDeployment("data", &count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, - c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL,scaling); err != nil { + c.ObjectMeta.Namespace, javaOptions, c.Spec.MasterJavaOptions, c.Spec.DataJavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, c.Spec.NodeSelector, c.Spec.Tolerations,scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err @@ -441,7 +442,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) // Create Master Nodes if err := p.k8sclient.CreateDataNodeDeployment("master", func() *int32 { i := int32(c.Spec.MasterNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, false); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions, c.Spec.MasterJavaOptions, c.Spec.DataJavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, c.Spec.NodeSelector, c.Spec.Tolerations, false); err != nil { logrus.Error("Error creating master node deployment ", err) return err @@ -464,7 +465,7 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster) if err := p.k8sclient.CreateDataNodeDeployment("data", func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), baseImage, c.Spec.Storage.StorageClass, c.Spec.DataDiskSize, resources, c.Spec.ImagePullSecrets, c.Spec.ImagePullPolicy, c.Spec.ServiceAccountName, c.ObjectMeta.Name, - c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL , scaling); err != nil { + c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, javaOptions, c.Spec.MasterJavaOptions, c.Spec.DataJavaOptions, c.Spec.UseSSL, c.Spec.Scheduler.ElasticURL, c.Spec.NodeSelector, c.Spec.Tolerations, scaling); err != nil { logrus.Error("Error creating data node deployment ", err) return err } From 572d6d65de81e0db588dd44bc8019af7c9e2ea07 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Sun, 13 Jan 2019 17:15:28 +0530 Subject: [PATCH 17/18] extra space removed. --- pkg/apis/elasticsearchoperator/v1/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/elasticsearchoperator/v1/cluster.go b/pkg/apis/elasticsearchoperator/v1/cluster.go index efa84be35..5d406ce74 100644 --- a/pkg/apis/elasticsearchoperator/v1/cluster.go +++ b/pkg/apis/elasticsearchoperator/v1/cluster.go @@ -78,7 +78,7 @@ type ClusterSpec struct { // Affinity (podAffinity, podAntiAffinity, nodeAffinity) will be applied to the Client nodes Affinity v1.Affinity `json:"affinity,omitempty"` - + // Zones specifies a map of key-value pairs. Defines which zones // to deploy persistent volumes for data nodes Zones []string `json:"zones,omitempty"` From 4b38fe2a3034ebc492f32681ec91b9f7dff0dc16 Mon Sep 17 00:00:00 2001 From: Naredula Janardhana Reddy Date: Sun, 27 Jan 2019 01:29:10 +0530 Subject: [PATCH 18/18] enabled verbose. --- Makefile | 2 +- pkg/k8sutil/scaling_test.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index de78859b7..8a49eb04c 100644 --- a/Makefile +++ b/Makefile @@ -34,4 +34,4 @@ helm-package: helm repo index --merge charts/index.yaml charts test: clean - go test $$(go list ./... | grep -v /vendor/) + go test -v $$(go list ./... | grep -v /vendor/) diff --git a/pkg/k8sutil/scaling_test.go b/pkg/k8sutil/scaling_test.go index 0c4a59d68..8bb9e3d88 100644 --- a/pkg/k8sutil/scaling_test.go +++ b/pkg/k8sutil/scaling_test.go @@ -2,7 +2,6 @@ package k8sutil import ( "testing" - "github.com/upmc-enterprises/elasticsearch-operator/pkg/k8sutil" ) @@ -16,21 +15,21 @@ const ( incase if testing function does not return the error then the unit test should fail. */ func Test_scaling_change_setting(t *testing.T) { - err := k8sutil.ES_change_settings(NonExistingIP, "1m", "request"); + err := ES_change_settings(NonExistingIP, "1m", "request"); if (err == nil){ t.Errorf("Scaling unittest change setting failed"); } } func Test_check_for_green(t *testing.T) { - err := k8sutil.ES_checkForGreen(NonExistingIP); + err := ES_checkForGreen(NonExistingIP); if (err == nil){ t.Errorf("Scaling unittest check_for_green failed"); } } func Test_check_for_nodeUp(t *testing.T) { - err := k8sutil.ES_checkForNodeUp(NonExistingIP, NonExistingPOD, 5); + err := ES_checkForNodeUp(NonExistingIP, NonExistingPOD, 5); if (err == nil){ t.Errorf("Scaling unittest check_for_nodeUp failed"); }