diff --git a/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew.go b/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew.go index fd89292edb5..dfc1dfbb1d4 100644 --- a/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew.go +++ b/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew.go @@ -18,6 +18,7 @@ import ( "github.com/sirupsen/logrus" "github.com/ugorji/go/codec" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" kruntime "k8s.io/apimachinery/pkg/runtime" @@ -37,6 +38,7 @@ type etcdrenew struct { secretNames []string mode string backupSecrets map[string][]byte + lastRevision int32 } var etcdOperatorControllerConditionsExpected = map[string]operatorv1.ConditionStatus{ @@ -79,7 +81,7 @@ func (e *etcdrenew) validate(ctx context.Context) error { func (e *etcdrenew) isRenewed(ctx context.Context) error { s := []steps.Step{ - steps.Condition(e.isRevisied, 30*time.Minute, true), + steps.Condition(e.isRevisied, 1*time.Minute, true), } _, err := steps.Run(ctx, e.log, 10*time.Second, s, nil) if err != nil { @@ -120,18 +122,17 @@ func (f *frontend) _postAdminOpenShiftClusterEtcdCertificateRenew(ctx context.Co return err } e := &etcdrenew{ - log: log, - k: k, - secretNames: nil, - mode: "renew", + log: log, + k: k, + secretNames: nil, + mode: "renew", + backupSecrets: make(map[string][]byte), + lastRevision: 0, } if err = e.validateClusterVersion(ctx); err != nil { return err } - if err = e.validate(ctx); err != nil { - return err - } // Fetch secretNames using nodeNames masterNodeNames, err := fetchNodeNames(ctx, k, log) @@ -148,15 +149,14 @@ func (f *frontend) _postAdminOpenShiftClusterEtcdCertificateRenew(ctx context.Co } } + if err = e.validate(ctx); err != nil { + return err + } // backup and delete etcd secrets if err = e.backupAndDelete(ctx); err != nil { return err } - // Calling Sleep method - e.log.Infoln("Entering sleep... 3mins") - time.Sleep(3 * time.Minute) - if err = e.isRenewed(ctx); err != nil { e.mode = "recovery" } else { @@ -197,12 +197,40 @@ func (e *etcdrenew) validateClusterVersion(ctx context.Context) error { return err } // ETCD ceritificates are autorotated by the operator when close to expiry for cluster running 4.9+ - if clusterVersion.Lt(version.NewVersion(4, 9)) { + if !clusterVersion.Lt(version.NewVersion(4, 9)) { return api.NewCloudError(http.StatusForbidden, api.CloudErrorCodeForbidden, "", "etcd certificate renewal is not needed for cluster running version 4.9+") } return nil } +func fetchNodeNames(ctx context.Context, k adminactions.KubeActions, log *logrus.Entry) ([]string, error) { + var masterNodeNames []string + var u unstructured.Unstructured + var nodes corev1.NodeList + + log.Infoln("fetching node names") + + nodeList, err := k.KubeList(ctx, "node", "") + if err != nil { + return nil, err + } + if err = json.Unmarshal(nodeList, &u); err != nil { + return nil, err + } + err = kruntime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &nodes) + if err != nil { + return nil, err + } + + for _, node := range nodes.Items { + if _, ok := node.ObjectMeta.Labels["node-role.kubernetes.io/master"]; ok { + masterNodeNames = append(masterNodeNames, node.ObjectMeta.Name) + continue + } + } + return masterNodeNames, nil +} + func (e *etcdrenew) validateEtcdOperatorControllersState(ctx context.Context) error { e.log.Infoln("validating etcdOperator Controllers state now") rawEtcd, err := e.k.KubeGet(ctx, "Etcd", "", "cluster") @@ -218,30 +246,15 @@ func (e *etcdrenew) validateEtcdOperatorControllersState(ctx context.Context) er if _, ok := etcdOperatorControllerConditionsExpected[c.Type]; !ok { continue } - if etcdOperatorControllerConditionsExpected[c.Type] != c.Status && e.mode == "renewed" { + if etcdOperatorControllerConditionsExpected[c.Type] != c.Status { return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "%s is in state %s, quiting.", c.Type, c.Status) } } - return nil -} - -func (e *etcdrenew) isRevisied(ctx context.Context) (bool, error) { - isAtRevision := true - rawEtcd, err := e.k.KubeGet(ctx, "Etcd", "", "cluster") - if err != nil { - return false, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + if e.mode == "renew" { + e.lastRevision = etcd.Status.LatestAvailableRevision } - etcd := &operatorv1.Etcd{} - err = codec.NewDecoderBytes(rawEtcd, &codec.JsonHandle{}).Decode(etcd) - if err != nil { - return false, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", fmt.Sprintf("failed to decode etcd object, %s", err.Error())) - } - for _, s := range etcd.Status.NodeStatuses { - if s.CurrentRevision != etcd.Status.LatestAvailableRevision { - isAtRevision = false - } - } - return isAtRevision, nil + + return nil } func (e *etcdrenew) validateEtcdOperatorState(ctx context.Context) error { @@ -259,39 +272,13 @@ func (e *etcdrenew) validateEtcdOperatorState(ctx context.Context) error { if _, ok := etcdOperatorConditionsExpected[c.Type]; !ok { continue } - if etcdOperatorConditionsExpected[c.Type] != c.Status && e.mode == "renewed" { + if etcdOperatorConditionsExpected[c.Type] != c.Status { return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Etcd Operator is not in expected state, quiting.") } } return nil } -func fetchNodeNames(ctx context.Context, k adminactions.KubeActions, log *logrus.Entry) ([]string, error) { - var masterNodeNames []string - var u unstructured.Unstructured - var nodes corev1.NodeList - - nodeList, err := k.KubeList(ctx, "node", "") - if err != nil { - return nil, err - } - if err = json.Unmarshal(nodeList, &u); err != nil { - return nil, err - } - err = kruntime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &nodes) - if err != nil { - return nil, err - } - - for _, node := range nodes.Items { - if _, ok := node.ObjectMeta.Labels["node-role.kubernetes.io/master"]; ok { - masterNodeNames = append(masterNodeNames, node.ObjectMeta.Name) - continue - } - } - return masterNodeNames, nil -} - func (e *etcdrenew) validateEtcdCertsExistsAndExpiry(ctx context.Context) error { e.log.Infoln("validating etcd certs exists, not expired but are not close to expiry") for _, secretname := range e.secretNames { @@ -313,11 +300,11 @@ func (e *etcdrenew) validateEtcdCertsExistsAndExpiry(ctx context.Context) error if err != nil { return err } - if !utilcert.IsLessThanMinimumDuration(certData[0], utilcert.DefaultMinDurationPercent) && e.mode == "renewed" { - return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "secret %s is not near expiry, quitting", secretname) + if !utilcert.IsLessThanMinimumDuration(certData[0], utilcert.DefaultMinDurationPercent) && e.mode != "renewed" { + return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "secret %s is not near expiry, quitting.", secretname) } if utilcert.IsCertExpired(certData[0]) { - return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "secret %s is already expired, quitting", secretname) + return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "secret %s is already expired, quitting.", secretname) } } @@ -332,6 +319,22 @@ func (e *etcdrenew) backupEtcdSecrets(ctx context.Context) error { if err != nil { return err } + secret := &corev1.Secret{} + err = codec.NewDecoderBytes(data, &codec.JsonHandle{}).Decode(secret) + if err != nil { + return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", fmt.Sprintf("failed to decode secret, %s", err.Error())) + } + secret.CreationTimestamp = metav1.Time{ + Time: time.Now(), + } + secret.ResourceVersion = "" + secret.SelfLink = "" + secret.UID = "" + + err = codec.NewEncoderBytes(&data, &codec.JsonHandle{}).Encode(secret) + if err != nil { + return api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", fmt.Sprintf("failed to encode secret, %s", err.Error())) + } e.backupSecrets[secretname] = data } return nil @@ -349,6 +352,28 @@ func (e *etcdrenew) deleteEtcdSecrets(ctx context.Context) error { return nil } +func (e *etcdrenew) isRevisied(ctx context.Context) (bool, error) { + isAtRevision := true + rawEtcd, err := e.k.KubeGet(ctx, "Etcd", "", "cluster") + if err != nil { + return false, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", err.Error()) + } + etcd := &operatorv1.Etcd{} + err = codec.NewDecoderBytes(rawEtcd, &codec.JsonHandle{}).Decode(etcd) + if err != nil { + return false, api.NewCloudError(http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", fmt.Sprintf("failed to decode etcd object, %s", err.Error())) + } + if e.lastRevision != etcd.Status.LatestAvailableRevision { + return false, nil + } + for _, s := range etcd.Status.NodeStatuses { + if s.CurrentRevision != etcd.Status.LatestAvailableRevision { + isAtRevision = false + } + } + return isAtRevision, nil +} + func (e *etcdrenew) recoverEtcdSecrets(ctx context.Context) error { e.log.Infoln("recovering etcd secrets now") for secretname, data := range e.backupSecrets { diff --git a/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go b/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go new file mode 100644 index 00000000000..ec5b1e4fc6b --- /dev/null +++ b/pkg/frontend/admin_openshiftcluster_etcdcertificaterenew_test.go @@ -0,0 +1,556 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/golang/mock/gomock" + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/sirupsen/logrus" + "github.com/ugorji/go/codec" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/frontend/adminactions" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions" + utiltls "github.com/Azure/ARO-RP/pkg/util/tls" +) + +func TestAdminEtcdCertificateRenew(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + ctx := context.Background() + nodes := &corev1.NodeList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + }, + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-0", + Labels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-1", + Labels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "master-2", + Labels: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + }, + }, + }, + } + + type test struct { + name string + resourceID string + version *configv1.ClusterVersion + etcdoperator *operatorv1.Etcd + etcdoperatorRevisied *operatorv1.Etcd + etcdCO *configv1.ClusterOperator + notBefore time.Time + notAfter time.Time + mocks func(*test, *mock_adminactions.MockKubeActions) + wantStatusCode int + wantResponse []byte + wantError string + } + + for _, tt := range []*test{ + { + name: "validate cluster version is <4.9", + resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID), + version: &configv1.ClusterVersion{ + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.11.44", + }, + }, + }, + }, + mocks: func(tt *test, k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeGet(gomock.Any(), "clusterversion", "", "version"). + Return(encodeClusterVersion(t, tt.version), nil) + }, + wantStatusCode: http.StatusForbidden, + wantError: "403: Forbidden: : etcd certificate renewal is not needed for cluster running version 4.9+", + }, + { + name: "validate etcd operator controller status", + resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID), + version: &configv1.ClusterVersion{ + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.8.11", + }, + }, + }, + }, + etcdoperator: &operatorv1.Etcd{ + Status: operatorv1.EtcdStatus{ + StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{ + { + Type: "EtcdCertSignerControllerDegraded", + Status: operatorv1.ConditionTrue, + }, + }, + }, + }, + }, + }, + mocks: func(tt *test, k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeGet(gomock.Any(), "clusterversion", "", "version").MaxTimes(1). + Return(encodeClusterVersion(t, tt.version), nil) + k.EXPECT(). + KubeList(gomock.Any(), "node", ""). + Return(encodeNodeList(t, nodes), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Etcd", "", "cluster"). + Return(encodeEtcdOperatorController(t, tt.etcdoperator), nil) + }, + wantStatusCode: http.StatusInternalServerError, + wantError: "500: InternalServerError: : EtcdCertSignerControllerDegraded is in state True, quiting.", + }, + { + name: "validate etcd cluster operator status", + resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID), + version: &configv1.ClusterVersion{ + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.8.11", + }, + }, + }, + }, + etcdoperator: &operatorv1.Etcd{ + Status: operatorv1.EtcdStatus{ + StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{ + { + Type: "EtcdCertSignerControllerDegraded", + Status: operatorv1.ConditionFalse, + }, + }, + }, + }, + }, + }, + etcdCO: &configv1.ClusterOperator{ + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionTrue, + }, + }, + }, + }, + mocks: func(tt *test, k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeGet(gomock.Any(), "clusterversion", "", "version").MaxTimes(1). + Return(encodeClusterVersion(t, tt.version), nil) + k.EXPECT(). + KubeList(gomock.Any(), "node", ""). + Return(encodeNodeList(t, nodes), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Etcd", "", "cluster").MaxTimes(1). + Return(encodeEtcdOperatorController(t, tt.etcdoperator), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "clusteroperator", "", "etcd"). + Return(encodeEtcdOperator(t, tt.etcdCO), nil) + + }, + wantStatusCode: http.StatusInternalServerError, + wantError: "500: InternalServerError: : Etcd Operator is not in expected state, quiting.", + }, + { + name: "validate if etcd certificates are not near expiry", + resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID), + version: &configv1.ClusterVersion{ + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.8.11", + }, + }, + }, + }, + etcdoperator: &operatorv1.Etcd{ + Status: operatorv1.EtcdStatus{ + StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{ + { + Type: "EtcdCertSignerControllerDegraded", + Status: operatorv1.ConditionFalse, + }, + }, + }, + }, + }, + }, + etcdCO: &configv1.ClusterOperator{ + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + }, + }, + }, + }, + notBefore: time.Now(), + notAfter: time.Now().AddDate(3, 0, 0), + mocks: func(tt *test, k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeGet(gomock.Any(), "clusterversion", "", "version").MaxTimes(1). + Return(encodeClusterVersion(t, tt.version), nil) + k.EXPECT(). + KubeList(gomock.Any(), "node", ""). + Return(encodeNodeList(t, nodes), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Etcd", "", "cluster").MaxTimes(1). + Return(encodeEtcdOperatorController(t, tt.etcdoperator), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "clusteroperator", "", "etcd"). + Return(encodeEtcdOperator(t, tt.etcdCO), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Secret", namespaceEtcds, gomock.Any()). + Return(CreateCertSecret(t, tt.notBefore, tt.notAfter), nil) + }, + wantStatusCode: http.StatusInternalServerError, + wantError: "500: InternalServerError: : secret etcd-peer-master-0 is not near expiry, quitting.", + }, + { + name: "validate if etcd certificates are expired", + resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID), + version: &configv1.ClusterVersion{ + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.8.11", + }, + }, + }, + }, + etcdoperator: &operatorv1.Etcd{ + Status: operatorv1.EtcdStatus{ + StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{ + { + Type: "EtcdCertSignerControllerDegraded", + Status: operatorv1.ConditionFalse, + }, + }, + }, + }, + }, + }, + etcdCO: &configv1.ClusterOperator{ + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + }, + }, + }, + }, + notBefore: time.Now(), + notAfter: time.Now().Add(-10 * time.Minute), + mocks: func(tt *test, k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeGet(gomock.Any(), "clusterversion", "", "version").MaxTimes(1). + Return(encodeClusterVersion(t, tt.version), nil) + k.EXPECT(). + KubeList(gomock.Any(), "node", ""). + Return(encodeNodeList(t, nodes), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Etcd", "", "cluster").MaxTimes(1). + Return(encodeEtcdOperatorController(t, tt.etcdoperator), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "clusteroperator", "", "etcd"). + Return(encodeEtcdOperator(t, tt.etcdCO), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Secret", namespaceEtcds, gomock.Any()). + Return(CreateCertSecret(t, tt.notBefore, tt.notAfter), nil) + }, + wantStatusCode: http.StatusInternalServerError, + wantError: "500: InternalServerError: : secret etcd-peer-master-0 is already expired, quitting.", + }, + { + name: "delete etcd secrets", + resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID), + version: &configv1.ClusterVersion{ + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + { + State: configv1.CompletedUpdate, + Version: "4.8.11", + }, + }, + }, + }, + etcdoperator: &operatorv1.Etcd{ + Status: operatorv1.EtcdStatus{ + StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{ + { + Type: "EtcdCertSignerControllerDegraded", + Status: operatorv1.ConditionFalse, + }, + }, + }, + LatestAvailableRevision: 1, + NodeStatuses: []operatorv1.NodeStatus{ + { + NodeName: "master-0", + CurrentRevision: 1, + }, + }, + }, + }, + }, + etcdoperatorRevisied: &operatorv1.Etcd{ + Status: operatorv1.EtcdStatus{ + StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + Conditions: []operatorv1.OperatorCondition{ + { + Type: "EtcdCertSignerControllerDegraded", + Status: operatorv1.ConditionFalse, + }, + }, + }, + LatestAvailableRevision: 2, + NodeStatuses: []operatorv1.NodeStatus{ + { + NodeName: "master-0", + CurrentRevision: 2, + }, + }, + }, + }, + }, + etcdCO: &configv1.ClusterOperator{ + Status: configv1.ClusterOperatorStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + }, + }, + }, + }, + notBefore: time.Now().AddDate(-2, -8, 0), + notAfter: time.Now().Add((1 * time.Hour)), + mocks: func(tt *test, k *mock_adminactions.MockKubeActions) { + k.EXPECT(). + KubeGet(gomock.Any(), "clusterversion", "", "version").MaxTimes(1). + Return(encodeClusterVersion(t, tt.version), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Etcd", "", "cluster").MaxTimes(1). + Return(encodeEtcdOperatorController(t, tt.etcdoperator), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "clusteroperator", "", "etcd").AnyTimes(). + Return(encodeEtcdOperator(t, tt.etcdCO), nil) + k.EXPECT(). + KubeList(gomock.Any(), "node", "").MaxTimes(1). + Return(encodeNodeList(t, nodes), nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Secret", namespaceEtcds, gomock.Any()).AnyTimes(). + Return(CreateCertSecret(t, tt.notBefore, tt.notAfter), nil) + d := k.EXPECT(). + KubeDelete(gomock.Any(), "Secret", namespaceEtcds, gomock.Any(), false, nil).MaxTimes(9). + Return(nil) + k.EXPECT(). + KubeGet(gomock.Any(), "Etcd", "", "cluster").AnyTimes().After(d). + Return(encodeEtcdOperatorController(t, tt.etcdoperatorRevisied), nil) + }, + wantStatusCode: http.StatusOK, + wantError: "", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions() + defer ti.done() + + k := mock_adminactions.NewMockKubeActions(ti.controller) + tt.mocks(tt, k) + + f, err := NewFrontend(ctx, + ti.audit, + ti.log, + ti.env, + ti.asyncOperationsDatabase, + ti.clusterManagerDatabase, + ti.openShiftClustersDatabase, + ti.subscriptionsDatabase, + nil, + api.APIs, + &noop.Noop{}, + &noop.Noop{}, + nil, + nil, + func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) { + return k, nil + }, + nil, + nil) + if err != nil { + t.Fatal(err) + } + + ti.fixture.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(tt.resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: tt.resourceID, + Name: "resourceName", + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + }, + }) + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err = ti.buildFixtures(nil) + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + + resp, b, err := ti.request(http.MethodPost, + fmt.Sprintf("https://server/admin%s/etcdcertificaterenew", tt.resourceID), + nil, nil) + if err != nil { + t.Fatal(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} + +func encodeClusterVersion(t *testing.T, version *configv1.ClusterVersion) []byte { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(version) + if err != nil { + t.Fatalf("%s failed to encode version, %s", t.Name(), err.Error()) + } + return buf.Bytes() +} + +func encodeEtcdOperatorController(t *testing.T, etcd *operatorv1.Etcd) []byte { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(etcd) + if err != nil { + t.Fatalf("%s failed to encode etcd operator, %s", t.Name(), err.Error()) + } + return buf.Bytes() +} + +func encodeEtcdOperator(t *testing.T, etcdCO *configv1.ClusterOperator) []byte { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(etcdCO) + if err != nil { + t.Fatalf("%s failed to encode etcd operator, %s", t.Name(), err.Error()) + } + return buf.Bytes() +} + +func encodeNodeList(t *testing.T, nodes *corev1.NodeList) []byte { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(nodes) + if err != nil { + t.Fatalf("%s failed to encode etcd cluster operator, %s", t.Name(), err.Error()) + } + return buf.Bytes() +} + +func encodeSecret(t *testing.T, secret *corev1.Secret) []byte { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(secret) + if err != nil { + t.Fatalf("%s failed to encode etcd secret, %s", t.Name(), err.Error()) + } + return buf.Bytes() +} + +func tweakTemplateFn(notBefore time.Time, notAfter time.Time) func(*x509.Certificate) { + return func(template *x509.Certificate) { + template.NotBefore = notBefore + template.NotAfter = notAfter + } +} + +func CreateCertSecret(t *testing.T, notBefore time.Time, notAfter time.Time) []byte { + secretname := "etcd-cert" + _, cert, err := utiltls.GenerateTestKeyAndCertificate(secretname, nil, nil, false, false, tweakTemplateFn(notBefore, notAfter)) + if err != nil { + t.Fatal(err) + } + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretname, + Namespace: "openshift-etcd", + }, + Data: map[string][]byte{ + corev1.TLSCertKey: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert[0].Raw}), + }, + Type: corev1.SecretTypeTLS, + } + return encodeSecret(t, secret) +}