Update ARO operator Azure auth scheme to use a DefaultAzureCredential (…

…#3274)

* Update the cluster authorizer to use a DefaultAzureCredential

* Update the ARO operator to set and use DefaultAzureCredential via env vars

* Add a CredentialsRequest to the ARO operator deployment

* Restart the ARO operator upon `az aro update`

* Removed now unused AzCredentials function

* Changed ARO operator deployment wait time during `az aro update` from
  20 minutes -> 5 minutes

* Refactor CliWithApply to generalize to different object types

* Updated Restart in pkg/util/kubernetes to use server-side apply
* Updated Restart in pkg/operator/deploy to only return an error after
  at least attempting to restart all of the deployments passed in

* E2E test for ARO operator master deployment's restart upon cluster update

* Wait for the ARO operator's CredentialsRequest to be reconciled before
restarting

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/securecookie v1.1.1
 	github.com/gorilla/sessions v1.2.1
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/itchyny/gojq v0.12.13
 	github.com/jewzaam/go-cosmosdb v0.0.0-20230924011506-8f8942a01991
 	github.com/jongio/azidext/go/azidext v0.5.0
@@ -56,6 +57,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
 	github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
 	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
+	github.com/openshift/cloud-credential-operator v0.0.0-00010101000000-000000000000
 	github.com/openshift/console-operator v0.0.0-20220407014945-45d37e70e0c2
 	github.com/openshift/hive/apis v0.0.0
 	github.com/openshift/library-go v0.0.0-20220525173854-9b950a41acdc
@@ -171,7 +173,6 @@ require (
 	github.com/gorilla/schema v1.2.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/itchyny/timefmt-go v0.1.5 // indirect

go.sum

@@ -675,6 +675,8 @@ github.com/openshift/build-machinery-go v0.0.0-20210115170933-e575b44a7a94/go.mo
 github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/client-go v0.0.0-20220603133046-984ee5ebedcf h1:gAYYPWVduONFJ6yuczLleApk0nEH3W0GgxDX2+O+B9E=
 github.com/openshift/client-go v0.0.0-20220603133046-984ee5ebedcf/go.mod h1:eDO5QeVi2IiXmDwB0e2z1DpAznWroZKe978pzZwFBzg=
+github.com/openshift/cloud-credential-operator v0.0.0-20200316201045-d10080b52c9e h1:2gyl9UVyjHSWzdS56KUXxQwIhENbq2x2olqoMQSA/C8=
+github.com/openshift/cloud-credential-operator v0.0.0-20200316201045-d10080b52c9e/go.mod h1:iPn+uhIe7nkP5BMHe2QnbLtg5m/AIQ1xvz9s3cig5ss=
 github.com/openshift/cluster-api-provider-azure v0.1.0-alpha.3.0.20210626224711-5d94c794092f h1:rQwvVLPZfM5o0USkVY6mrAyJwzMUkhjn9Wz2D5vX81k=
 github.com/openshift/cluster-api-provider-azure v0.1.0-alpha.3.0.20210626224711-5d94c794092f/go.mod h1:GR+ocB8I+Z7JTSBdO+DMu/diBfH66lRlRpnc1KWysUM=
 github.com/openshift/console-operator v0.0.0-20220318130441-e44516b9c315 h1:zmwv8TgbOgZ5QoaPhLdOivqg706Z+VyuPs703jNMdrE=

pkg/cluster/arooperator.go

@@ -5,6 +5,17 @@ package cluster
 
 import (
 	"context"
+
+	cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	CredentialsRequestGroupVersionResource = schema.GroupVersionResource{
+		Group:    cloudcredentialv1.SchemeGroupVersion.Group,
+		Version:  cloudcredentialv1.SchemeGroupVersion.Version,
+		Resource: "credentialsrequests",
+	}
 )
 
 func (m *manager) isIngressProfileAvailable() bool {
@@ -53,3 +64,7 @@ func (m *manager) ensureAROOperatorRunningDesiredVersion(ctx context.Context) (b
 func (m *manager) renewMDSDCertificate(ctx context.Context) error {
 	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
 }
+
+func (m *manager) restartAROOperatorMaster(ctx context.Context) error {
+	return m.aroOperatorDeployer.Restart(ctx, []string{"aro-operator-master"})
+}

pkg/cluster/cluster.go

@@ -18,6 +18,7 @@ import (
 	mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/Azure/ARO-RP/pkg/api"
@@ -89,6 +90,7 @@ type manager struct {
 	graph   graph.Manager
 
 	kubernetescli    kubernetes.Interface
+	dynamiccli       dynamic.Interface
 	extensionscli    extensionsclient.Interface
 	maocli           machineclient.Interface
 	mcocli           mcoclient.Interface

pkg/cluster/clusterserviceprincipal.go

@@ -4,12 +4,14 @@ package cluster
 // Licensed under the Apache License 2.0.
 
 import (
+	"bytes"
 	"context"
 	"strings"
 	"time"
 
 	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
 	"github.com/ghodss/yaml"
+	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	applyv1 "k8s.io/client-go/applyconfigurations/core/v1"
@@ -77,47 +79,70 @@ func (m *manager) createOrUpdateClusterServicePrincipalRBAC(ctx context.Context)
 	return nil
 }
 
-func (m *manager) updateAROSecret(ctx context.Context) error {
-	var changed bool
+// cloudConfigSecretFromChanges takes in the kube-system/azure-cloud-provider Secret and a map
+// containing cloud-config data. If the cloud-config data in cf is different from what's currently
+// in the Secret, cloudConfigSecretFromChanges updates and returns the Secret. Otherwise, it returns nil.
+func cloudConfigSecretFromChanges(secret *corev1.Secret, cf map[string]interface{}) (*corev1.Secret, error) {
+	data, err := yaml.Marshal(cf)
+	if err != nil {
+		return nil, err
+	}
+
+	if !bytes.Equal(secret.Data["cloud-config"], data) {
+		secret.Data["cloud-config"] = data
+		return secret, nil
+	}
+
+	return nil, nil
+}
+
+// servicePrincipalUpdated checks whether the CSP has been updated by comparing the cluster doc's
+// ServicePrincipalProfile to the contents of the kube-system/azure-cloud-provider Secret. If the CSP
+// has changed, it returns a new corev1.Secret to use to update the Secret to match
+// what's in the cluster doc.
+func (m *manager) servicePrincipalUpdated(ctx context.Context) (*corev1.Secret, error) {
 	spp := m.doc.OpenShiftCluster.Properties.ServicePrincipalProfile
-	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		//data:
-		// cloud-config: <base64 map[string]string with keys 'aadClientId' and 'aadClientSecret'>
-		secret, err := m.kubernetescli.CoreV1().Secrets("kube-system").Get(ctx, "azure-cloud-provider", metav1.GetOptions{})
-		if err != nil {
-			if kerrors.IsNotFound(err) { // we are not in control if secret is not present
-				return nil
-			}
-			return err
+	//data:
+	// cloud-config: <base64 map[string]string with keys 'aadClientId' and 'aadClientSecret'>
+	secret, err := m.kubernetescli.CoreV1().Secrets("kube-system").Get(ctx, "azure-cloud-provider", metav1.GetOptions{})
+	if err != nil {
+		if kerrors.IsNotFound(err) { // we are not in control if secret is not present
+			return nil, nil
 		}
+		return nil, err
+	}
 
-		var cf map[string]interface{}
-		if secret != nil && secret.Data != nil {
-			err = yaml.Unmarshal(secret.Data["cloud-config"], &cf)
-			if err != nil {
-				return err
-			}
-			if val, ok := cf["aadClientId"].(string); ok {
-				if val != spp.ClientID {
-					cf["aadClientId"] = spp.ClientID
-					changed = true
-				}
+	var cf map[string]interface{}
+	if secret != nil && secret.Data != nil {
+		err = yaml.Unmarshal(secret.Data["cloud-config"], &cf)
+		if err != nil {
+			return nil, err
+		}
+		if val, ok := cf["aadClientId"].(string); ok {
+			if val != spp.ClientID {
+				cf["aadClientId"] = spp.ClientID
 			}
-			if val, ok := cf["aadClientSecret"].(string); ok {
-				if val != string(spp.ClientSecret) {
-					cf["aadClientSecret"] = spp.ClientSecret
-					changed = true
-				}
+		}
+		if val, ok := cf["aadClientSecret"].(string); ok {
+			if val != string(spp.ClientSecret) {
+				cf["aadClientSecret"] = spp.ClientSecret
 			}
 		}
+	}
 
-		if changed {
-			data, err := yaml.Marshal(cf)
-			if err != nil {
-				return err
-			}
-			secret.Data["cloud-config"] = data
+	return cloudConfigSecretFromChanges(secret, cf)
+}
 
+func (m *manager) updateAROSecret(ctx context.Context) error {
+	var changed bool
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		secret, err := m.servicePrincipalUpdated(ctx)
+		changed = secret != nil
+		if err != nil {
+			return err
+		}
+
+		if changed {
 			_, err = m.kubernetescli.CoreV1().Secrets("kube-system").Update(ctx, secret, metav1.UpdateOptions{})
 			if err != nil {
 				return err