Maintenance Signals

pkg/api/admin/openshiftcluster.go

@@ -34,6 +34,7 @@ type OpenShiftClusterProperties struct {
 	FailedProvisioningState         ProvisioningState       `json:"failedProvisioningState,omitempty"`
 	LastAdminUpdateError            string                  `json:"lastAdminUpdateError,omitempty"`
 	MaintenanceTask                 MaintenanceTask         `json:"maintenanceTask,omitempty" mutable:"true"`
+	MaintenanceState                MaintenanceState        `json:"maintenanceState,omitempty" mutable:"true"`
 	OperatorFlags                   OperatorFlags           `json:"operatorFlags,omitempty" mutable:"true"`
 	OperatorVersion                 string                  `json:"operatorVersion,omitempty" mutable:"true"`
 	CreatedAt                       time.Time               `json:"createdAt,omitempty"`
@@ -81,9 +82,21 @@ const (
 type MaintenanceTask string
 
 const (
-	MaintenanceTaskEverything MaintenanceTask = "Everything"
-	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
-	MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
+	MaintenanceTaskEverything  MaintenanceTask = "Everything"
+	MaintenanceTaskOperator    MaintenanceTask = "OperatorUpdate"
+	MaintenanceTaskRenewCerts  MaintenanceTask = "CertificatesRenewal"
+	MaintenanceTaskStateUpdate MaintenanceTask = "MaintenanceStateUpdate"
+)
+
+// MaintenanceState represents a maintenance state
+type MaintenanceState string
+
+// MaintenanceState constants
+const (
+	MaintenanceStateNone      MaintenanceState = "None"
+	MaintenanceStatePending   MaintenanceState = "Pending"
+	MaintenanceStatePlanned   MaintenanceState = "Planned"
+	MaintenanceStateUnplanned MaintenanceState = "Unplanned"
 )
 
 // Operator feature flags

pkg/api/admin/openshiftcluster_convert.go

@@ -26,6 +26,7 @@ func (c openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfac
 			FailedProvisioningState: ProvisioningState(oc.Properties.FailedProvisioningState),
 			LastAdminUpdateError:    oc.Properties.LastAdminUpdateError,
 			MaintenanceTask:         MaintenanceTask(oc.Properties.MaintenanceTask),
+			MaintenanceState:        MaintenanceState(oc.Properties.MaintenanceState),
 			OperatorFlags:           OperatorFlags(oc.Properties.OperatorFlags),
 			OperatorVersion:         oc.Properties.OperatorVersion,
 			CreatedAt:               oc.Properties.CreatedAt,
@@ -173,6 +174,7 @@ func (c openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShif
 	out.Properties.FailedProvisioningState = api.ProvisioningState(oc.Properties.FailedProvisioningState)
 	out.Properties.LastAdminUpdateError = oc.Properties.LastAdminUpdateError
 	out.Properties.MaintenanceTask = api.MaintenanceTask(oc.Properties.MaintenanceTask)
+	out.Properties.MaintenanceState = api.MaintenanceState(oc.Properties.MaintenanceState)
 	out.Properties.OperatorFlags = api.OperatorFlags(oc.Properties.OperatorFlags)
 	out.Properties.OperatorVersion = oc.Properties.OperatorVersion
 	out.Properties.CreatedBy = oc.Properties.CreatedBy

pkg/api/admin/openshiftcluster_validatestatic.go

@@ -29,9 +29,47 @@ func (sv openShiftClusterStaticValidator) validateDelta(oc, current *OpenShiftCl
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, err.Target, err.Message)
 	}
 
-	if !(oc.Properties.MaintenanceTask == "" || oc.Properties.MaintenanceTask == MaintenanceTaskEverything || oc.Properties.MaintenanceTask == MaintenanceTaskOperator || oc.Properties.MaintenanceTask == MaintenanceTaskRenewCerts) {
+	if invalidMaintenanceTask(oc.Properties.MaintenanceTask) {
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.")
 	}
 
+	if invalidMaintenanceState(oc.Properties.MaintenanceState) {
+		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceState", "Invalid enum parameter.")
+	}
+
 	return nil
 }
+
+func invalidMaintenanceTask(task MaintenanceTask) bool {
+	switch task {
+	case "":
+		fallthrough
+	case MaintenanceTaskEverything:
+		fallthrough
+	case MaintenanceTaskOperator:
+		fallthrough
+	case MaintenanceTaskRenewCerts:
+		fallthrough
+	case MaintenanceTaskStateUpdate:
+		return false
+	}
+
+	return true
+}
+
+func invalidMaintenanceState(state MaintenanceState) bool {
+	switch state {
+	case "":
+		fallthrough
+	case MaintenanceStateNone:
+		fallthrough
+	case MaintenanceStatePending:
+		fallthrough
+	case MaintenanceStatePlanned:
+		fallthrough
+	case MaintenanceStateUnplanned:
+		return false
+	}
+
+	return true
+}

pkg/api/admin/openshiftcluster_validatestatic_test.go

@@ -690,6 +690,85 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 			},
 			wantErr: "400: InvalidParameter: properties.maintenanceTask: Invalid enum parameter.",
 		},
+		{
+			name: "maintenanceState change to blank allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceState: MaintenanceStateNone,
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceState = ""
+			},
+		},
+		{
+			name: "maintenanceState change to none allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceState: "",
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceState = MaintenanceStateNone
+			},
+		},
+		{
+			name: "maintenanceState change to pending allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceState: MaintenanceStateNone,
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceState = MaintenanceStatePending
+			},
+		},
+		{
+			name: "maintenanceState change to planned allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceState: MaintenanceStateNone,
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceState = MaintenanceStatePlanned
+			},
+		},
+		{
+			name: "maintenanceState change to unplanned allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceState: MaintenanceStateNone,
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceState = MaintenanceStateUnplanned
+			},
+		},
+		{
+			name: "maintenanceState change to other values is disallowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceState: MaintenanceStateNone,
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceState = "adfasdfadf"
+			},
+			wantErr: "400: InvalidParameter: properties.maintenanceState: Invalid enum parameter.",
+		},
 	}
 
 	for _, tt := range tests {

pkg/api/defaults.go

@@ -46,6 +46,11 @@ func SetDefaults(doc *OpenShiftClusterDocument) {
 		if doc.OpenShiftCluster.Properties.NetworkProfile.PreconfiguredNSG == "" {
 			doc.OpenShiftCluster.Properties.NetworkProfile.PreconfiguredNSG = PreconfiguredNSGDisabled
 		}
+
+		// If there's no maintenance states, set to none
+		if doc.OpenShiftCluster.Properties.MaintenanceState == "" {
+			doc.OpenShiftCluster.Properties.MaintenanceState = MaintenanceStateNone
+		}
 	}
 }
 

pkg/api/openshiftcluster.go

@@ -100,6 +100,7 @@ type OpenShiftClusterProperties struct {
 	FailedProvisioningState ProvisioningState   `json:"failedProvisioningState,omitempty"`
 	LastAdminUpdateError    string              `json:"lastAdminUpdateError,omitempty"`
 	MaintenanceTask         MaintenanceTask     `json:"maintenanceTask,omitempty"`
+	MaintenanceState        MaintenanceState    `json:"maintenanceState,omitempty"`
 
 	// Operator feature/option flags
 	OperatorFlags   OperatorFlags `json:"operatorFlags,omitempty"`
@@ -175,9 +176,21 @@ const (
 type MaintenanceTask string
 
 const (
-	MaintenanceTaskEverything MaintenanceTask = "Everything"
-	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
-	MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
+	MaintenanceTaskEverything  MaintenanceTask = "Everything"
+	MaintenanceTaskOperator    MaintenanceTask = "OperatorUpdate"
+	MaintenanceTaskRenewCerts  MaintenanceTask = "CertificatesRenewal"
+	MaintenanceTaskStateUpdate MaintenanceTask = "MaintenanceStateUpdate"
+)
+
+// MaintenanceState represents a maintenance state
+type MaintenanceState string
+
+// MaintenanceState constants
+const (
+	MaintenanceStateNone      MaintenanceState = "None"
+	MaintenanceStatePending   MaintenanceState = "Pending"
+	MaintenanceStatePlanned   MaintenanceState = "Planned"
+	MaintenanceStateUnplanned MaintenanceState = "Unplanned"
 )
 
 // Cluster-scoped flags
@@ -192,6 +205,10 @@ func (t ProvisioningState) String() string {
 	return string(t)
 }
 
+func (t MaintenanceState) String() string {
+	return string(t)
+}
+
 // FipsValidatedModules determines if FIPS is used.
 type FipsValidatedModules string
 

pkg/cluster/adminupdate_test.go

@@ -252,6 +252,24 @@ func TestAdminUpdateSteps(t *testing.T) {
 				"[Action updateProvisionedBy-fm]",
 			},
 		},
+		{
+			name: "Maintenance State Update",
+			fixture: func() (*api.OpenShiftClusterDocument, bool) {
+				doc := baseClusterDoc()
+				doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
+				doc.OpenShiftCluster.Properties.MaintenanceTask = api.MaintenanceTaskStateUpdate
+				return doc, true
+			},
+			shouldRunSteps: []string{
+				"[Action initializeKubernetesClients-fm]",
+				"[Action ensureBillingRecord-fm]",
+				"[Action ensureDefaults-fm]",
+				"[AuthorizationRetryingAction fixupClusterSPObjectID-fm]",
+				"[Action fixInfraID-fm]",
+				"[Action startVMs-fm]",
+				"[Condition apiServersReady-fm, timeout 30m0s]",
+			},
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			doc, adoptViaHive := tt.fixture()

pkg/monitor/cluster/cluster.go

@@ -170,6 +170,7 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
 		mon.emitSummary,
 		mon.emitHiveRegistrationStatus,
 		mon.emitOperatorFlagsAndSupportBanner,
+		mon.emitClusterMaintenanceState,
 		mon.emitPrometheusAlerts, // at the end for now because it's the slowest/least reliable
 	} {
 		err = f(ctx)

pkg/monitor/cluster/clustermaintenancestate.go

@@ -0,0 +1,14 @@
+package cluster
+
+import "context"
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+func (mon *Monitor) emitClusterMaintenanceState(ctx context.Context) error {
+	mon.emitGauge("cluster.maintenanceState", 1, map[string]string{
+		"maintenanceState": mon.oc.Properties.MaintenanceState.String(),
+	})
+
+	return nil
+}

pkg/monitor/cluster/clustermaintenancestate_test.go

@@ -0,0 +1,41 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	mock_metrics "github.com/Azure/ARO-RP/pkg/util/mocks/metrics"
+)
+
+func TestEmitClusterMaintenanceState(t *testing.T) {
+	ctx := context.Background()
+
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	m := mock_metrics.NewMockEmitter(controller)
+
+	mon := &Monitor{
+		m: m,
+		oc: &api.OpenShiftCluster{
+			Properties: api.OpenShiftClusterProperties{
+				MaintenanceState: api.MaintenanceStateNone,
+			},
+		},
+	}
+
+	m.EXPECT().EmitGauge("cluster.maintenanceState", int64(1), map[string]string{
+		"maintenanceState": "None",
+	})
+
+	err := mon.emitClusterMaintenanceState(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+}