[Draft]: Spike: CSPL-3458: initial code changes for zeroD

api/v4/common_types.go

@@ -569,6 +569,11 @@ type PhaseInfo struct {
 	FailCount uint32 `json:"failCount,omitempty"`
 }
 
+type UpgradeStrategy struct {
+	WaitForHistSearchDomain            bool `json:"waitForHistSearchDrain,omitempty"`
+	EnableSearchRestartOnRemoteFailure bool `json:"enableSearchRestartOnRemoteFailure,omitempty"`
+}
+
 const (
 	// AppPkgDownloadPending indicates pending
 	AppPkgDownloadPending AppPhaseStatusType = 101

api/v4/searchheadcluster_types.go

@@ -50,6 +50,9 @@ type SearchHeadClusterSpec struct {
 
 	// Splunk Deployer Node Affinity
 	DeployerNodeAffinity *corev1.NodeAffinity `json:"deployerNodeAffinity,omitempty"`
+
+	// upgrade strategy for the search head cluster
+	UpgradeStrategy UpgradeStrategy `json:"upgradeStrategy,omitempty"`
 }
 
 // SearchHeadClusterMemberStatus is used to track the status of each search head cluster member
@@ -128,6 +131,8 @@ type SearchHeadClusterStatus struct {
 
 	// Auxillary message describing CR status
 	Message string `json:"message"`
+
+	UpgradePhase string `json:"upgradePhase"`
 }
 
 // SearchHeadCluster is the Schema for a Splunk Enterprise search head cluster

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
 	github.com/aws/aws-sdk-go v1.47.11
 	github.com/go-logr/logr v1.4.2
+	github.com/go-resty/resty/v2 v2.16.5
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/joho/godotenv v1.5.1

go.sum

@@ -131,6 +131,8 @@ github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXym
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
+github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=

pkg/splunk/client/enterprise.go

@@ -24,8 +24,8 @@
 	"regexp"
 	"strconv"
 	"strings"
-	"time"
 
+	"github.com/go-resty/resty/v2"
 	splcommon "github.com/splunk/splunk-operator/pkg/splunk/common"
 )
 
@@ -35,6 +35,35 @@
 	Do(*http.Request) (*http.Response, error)
 }
 
+// RestyClientWrapper is a wrapper around resty.Client to implement SplunkHTTPClient interface
+type RestyClientWrapper struct {
+	*resty.Client
+}
+
+// Do sends an HTTP request and returns an HTTP response, implementing the SplunkHTTPClient interface
+func (r *RestyClientWrapper) Do(req *http.Request) (*http.Response, error) {
+	restyReq := r.R()
+	restyReq.Method = req.Method
+	restyReq.URL = req.URL.String()
+	restyReq.Body = req.Body
+	restyReq.Header = req.Header
+
+	resp, err := restyReq.Send()
+	if err != nil {
+		return nil, err
+	}
+
+	httpResp := &http.Response{
+		Status:        resp.Status(),
+		StatusCode:    resp.StatusCode(),
+		Body:          io.NopCloser(strings.NewReader(resp.String())),
+		Header:        resp.Header(),
+		ContentLength: resp.Size(),
+	}
+
+	return httpResp, nil
+}
+
 // SplunkClient is a simple object used to send HTTP REST API requests
 type SplunkClient struct {
 	// https endpoint for management interface (e.g. "https://server:8089")
@@ -50,18 +79,35 @@
 	Client SplunkHTTPClient
 }
 
+// UpgradeMetrics holds search metrics for a search head during upgrade.
+type UpgradeMetrics struct {
+	ShortSearchSuccess int `json:"short_search_success"`
+	ShortSearchFailure int `json:"short_search_failure"`
+	TotalSearchSuccess int `json:"total_search_success"`
+	TotalSearchFailure int `json:"total_search_failure"`
+}
+
 // NewSplunkClient returns a new SplunkClient object initialized with a username and password.
 func NewSplunkClient(managementURI, username, password string) *SplunkClient {
+	client := &RestyClientWrapper{
+		Client: resty.New().
+			SetBaseURL(managementURI).
+			SetBasicAuth(username, password).
+			SetRetryCount(3).
+			SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}),
+	}
+
 	return &SplunkClient{
 		ManagementURI: managementURI,
 		Username:      username,
 		Password:      password,
-		Client: &http.Client{
-			Timeout: 5 * time.Second,
-			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // don't verify ssl certs
-			},
-		},
+		Client:        client,
+		//Client: &http.Client{
+		//	Timeout: 5 * time.Second,
+		//	Transport: &http.Transport{
+		//		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // don't verify ssl certs
+		//	},
+		//},
 	}
 }
 
@@ -935,3 +981,75 @@
 	expectedStatus := []int{200}
 	return c.Do(request, expectedStatus, nil)
 }
+
+// GetUpgradeSearchMetrics uses Resty to query the search head for upgrade search metrics.
+func (c *SplunkClient) GetUpgradeSearchMetrics() (*UpgradeMetrics, error) {
+	resp, err := c.Client.(*RestyClientWrapper).R().
+		SetResult(&UpgradeMetrics{}).
+		Get("/services/search/metrics")
+	if err != nil {
+		return nil, err
+	}
+	if resp.IsError() {
+		return nil, fmt.Errorf("error fetching upgrade metrics: %s", resp.Status())
+	}
+	metrics, ok := resp.Result().(*UpgradeMetrics)
+	if !ok {
+		return nil, fmt.Errorf("failed to parse upgrade metrics")
+	}
+	return metrics, nil
+}
+
+// InitShcUpgrade calls the endpoint to display the upgrade banner.
+func (c *SplunkClient) InitShcUpgrade() error {
+	resp, err := c.Client.(*RestyClientWrapper).R().
+		Post("/services/shcluster/captain/control/control/upgrade-init")
+	if err != nil {
+		return err
+	}
+	if resp.IsError() {
+		return fmt.Errorf("error initiating SHC upgrade: %s", resp.Status())
+	}
+	return nil
+}
+
+// FinalizeShcUpgrade calls the endpoint to remove the upgrade banner.
+func (c *SplunkClient) FinalizeShcUpgrade() error {
+	resp, err := c.Client.(*RestyClientWrapper).R().
+		Post("/services/shcluster/captain/control/control/upgrade-finalize")
+	if err != nil {
+		return err
+	}
+	if resp.IsError() {
+		return fmt.Errorf("error finalizing SHC upgrade: %s", resp.Status())
+	}
+	return nil
+}
+
+// SetManualDetentionMode puts the search head into manual detention mode.
+func (c *SplunkClient) SetManualDetentionMode() error {
+	resp, err := c.Client.(*RestyClientWrapper).R().
+		SetBody(map[string]bool{"mode": true}).
+		Post("/servicesNS/admin/search/shcluster/member/control/control/set_manual_detention")
+	if err != nil {
+		return err
+	}
+	if resp.IsError() {
+		return fmt.Errorf("error setting manual detention mode: %s", resp.Status())
+	}
+	return nil
+}
+
+// UnsetManualDetentionMode removes the search head from manual detention mode.
+func (c *SplunkClient) UnsetManualDetentionMode() error {
+	resp, err := c.Client.(*RestyClientWrapper).R().
+		SetBody(map[string]bool{"mode": false}).
+		Post("/servicesNS/admin/search/shcluster/member/control/control/set_manual_detention")
+	if err != nil {
+		return err
+	}
+	if resp.IsError() {
+		return fmt.Errorf("error unsetting manual detention mode: %s", resp.Status())
+	}
+	return nil
+}

pkg/splunk/enterprise/metrics/metrics.go

@@ -0,0 +1,53 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	UpgradeStartTime = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "splunk_upgrade_start_time",
+		Help: "Unix timestamp when the SHC upgrade started",
+	})
+	UpgradeEndTime = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "splunk_upgrade_end_time",
+		Help: "Unix timestamp when the SHC upgrade ended",
+	})
+	ShortSearchSuccessCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "splunk_short_search_success_total",
+			Help: "Total number of successful short searches per search head",
+		},
+		[]string{"sh_name"},
+	)
+	ShortSearchFailureCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "splunk_short_search_failure_total",
+			Help: "Total number of failed short searches per search head",
+		},
+		[]string{"sh_name"},
+	)
+	TotalSearchSuccessCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "splunk_total_search_success_total",
+			Help: "Total number of successful total searches per search head",
+		},
+		[]string{"sh_name"},
+	)
+	TotalSearchFailureCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "splunk_total_search_failure_total",
+			Help: "Total number of failed total searches per search head",
+		},
+		[]string{"sh_name"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(UpgradeStartTime)
+	prometheus.MustRegister(UpgradeEndTime)
+	prometheus.MustRegister(ShortSearchSuccessCounter)
+	prometheus.MustRegister(ShortSearchFailureCounter)
+	prometheus.MustRegister(TotalSearchSuccessCounter)
+	prometheus.MustRegister(TotalSearchFailureCounter)
+}