ARO-4373 generate keypair and oidc docs for miwi clusters

go.mod

@@ -80,6 +80,7 @@ require (
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.15.0
 	golang.org/x/tools v0.19.0
+	gopkg.in/square/go-jose.v2 v2.6.0
 	k8s.io/api v0.30.0
 	k8s.io/apiextensions-apiserver v0.25.0
 	k8s.io/apimachinery v0.30.0
@@ -265,7 +266,6 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

pkg/cluster/cluster.go

@@ -30,6 +30,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/metrics"
 	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
 	"github.com/Azure/ARO-RP/pkg/operator/deploy"
+	"github.com/Azure/ARO-RP/pkg/util/azblob"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
@@ -92,6 +93,7 @@ type manager struct {
 	storage storage.Manager
 	subnet  subnet.Manager
 	graph   graph.Manager
+	rpBlob  azblob.Manager
 
 	client           client.Client
 	kubernetescli    kubernetes.Interface
@@ -142,6 +144,11 @@ func New(ctx context.Context, log *logrus.Entry, _env env.Interface, db database
 		return nil, err
 	}
 
+	msiCredential, err := _env.NewMSITokenCredential()
+	if err != nil {
+		return nil, err
+	}
+
 	msiAuthorizer, err := _env.NewMSIAuthorizer(_env.Environment().ResourceManagerScope)
 	if err != nil {
 		return nil, err
@@ -169,6 +176,11 @@ func New(ctx context.Context, log *logrus.Entry, _env env.Interface, db database
 		return nil, err
 	}
 
+	rpBlob, err := azblob.NewManager(_env, _env.SubscriptionID(), msiCredential)
+	if err != nil {
+		return nil, err
+	}
+
 	return &manager{
 		log:                   log,
 		env:                   _env,
@@ -205,6 +217,7 @@ func New(ctx context.Context, log *logrus.Entry, _env env.Interface, db database
 		storage: storage,
 		subnet:  subnet.NewManager(_env.Environment(), r.SubscriptionID, fpAuthorizer),
 		graph:   graph.NewManager(_env, log, aead, storage),
+		rpBlob:  rpBlob,
 
 		installViaHive:                    installViaHive,
 		adoptViaHive:                      adoptByHive,

pkg/cluster/delete.go

@@ -441,6 +441,12 @@ func (m *manager) Delete(ctx context.Context) error {
 		return err
 	}
 
+	m.log.Printf("deleting OIDC configuration")
+	err = m.rpBlob.DeleteBlobContainer(ctx, m.env.ResourceGroup(), m.env.OIDCStorageAccountName(), env.OIDCBlobContainerPrefix+m.doc.ID)
+	if err != nil {
+		return err
+	}
+
 	m.log.Printf("deleting role assignments")
 	err = m.deleteRoleAssignments(ctx)
 	if err != nil {

pkg/cluster/deploybaseresources.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	azstorage "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
 	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	"github.com/Azure/go-autorest/autorest"
@@ -24,6 +26,7 @@ import (
 	apisubnet "github.com/Azure/ARO-RP/pkg/api/util/subnet"
 	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/util/arm"
+	"github.com/Azure/ARO-RP/pkg/util/oidcbuilder"
 	"github.com/Azure/ARO-RP/pkg/util/stringutils"
 )
 
@@ -35,6 +38,48 @@ func (m *manager) createDNS(ctx context.Context) error {
 	return m.dns.Create(ctx, m.doc.OpenShiftCluster)
 }
 
+func (m *manager) createOIDC(ctx context.Context) error {
+	// TODO: Uncomment below part after testing OIDC creation in different environments
+	// if m.doc.OpenShiftCluster.Properties.ServicePrincipalProfile == nil {
+	// 	return nil
+	// }
+
+	blobContainerName := env.OIDCBlobContainerPrefix + m.doc.ID
+
+	publicAccess := azstorage.PublicAccessNone
+	// Public access on OIDC Container needed for development environments because of no AFD avaialbility
+	if m.env.IsLocalDevelopmentMode() {
+		publicAccess = azstorage.PublicAccessBlob
+	}
+	err := m.rpBlob.CreateBlobContainer(ctx, m.env.ResourceGroup(), m.env.OIDCStorageAccountName(), blobContainerName, publicAccess)
+	if err != nil {
+		return err
+	}
+
+	oidcBuilder, err := oidcbuilder.NewOIDCBuilder(m.env.Environment().StorageEndpointSuffix, m.env.OIDCEndpoint(), m.env.OIDCStorageAccountName(), blobContainerName)
+	if err != nil {
+		return err
+	}
+
+	azBlobClient, err := m.rpBlob.GetAZBlobClient(oidcBuilder.GetBlobContainerURL(), &azblob.ClientOptions{})
+	if err != nil {
+		return err
+	}
+
+	err = oidcBuilder.EnsureOIDCDocs(ctx, blobContainerName, azBlobClient)
+	if err != nil {
+		return err
+	}
+
+	m.doc, err = m.db.PatchWithLease(ctx, m.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
+		doc.OpenShiftCluster.Properties.ClusterProfile.OIDCIssuer = api.OIDCIssuer(oidcBuilder.GetEndpointUrl())
+		doc.OpenShiftCluster.Properties.ClusterProfile.BoundServiceAccountSigningKey = api.SecureString(oidcBuilder.GetPrivateKey())
+		return nil
+	})
+
+	return err
+}
+
 func (m *manager) ensureInfraID(ctx context.Context) (err error) {
 	if m.doc.OpenShiftCluster.Properties.InfraID != "" {
 		return err

pkg/cluster/install.go

@@ -280,6 +280,7 @@ func (m *manager) bootstrap() []steps.Step {
 		steps.Action(m.populateMTUSize),
 
 		steps.Action(m.createDNS),
+		steps.Action(m.createOIDC),
 		steps.Action(m.initializeClusterSPClients), // must run before clusterSPObjectID
 
 		// TODO: this relies on an authorizer that isn't exposed in the manager

pkg/env/env.go

@@ -58,6 +58,7 @@ const (
 	ServiceKeyvaultSuffix            = "-svc"
 	RPPrivateEndpointPrefix          = "rp-pe-"
 	ProxyHostName                    = "PROXY_HOSTNAME"
+	OIDCBlobContainerPrefix          = "oic-"
 )
 
 // Interface is clunky and somewhat legacy and only used in the RP codebase (not

pkg/util/azblob/generate.go

@@ -0,0 +1,8 @@
+package azblob
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate rm -rf ../mocks/$GOPACKAGE
+//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/$GOPACKAGE Manager
+//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../mocks/$GOPACKAGE/$GOPACKAGE.go

pkg/util/azblob/manager.go

@@ -0,0 +1,102 @@
+package azblob
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	azstorage "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armstorage"
+)
+
+type Manager interface {
+	CreateBlobContainer(ctx context.Context, resourceGroup string, account string, container string, publicAccess azstorage.PublicAccess) error
+	DeleteBlobContainer(ctx context.Context, resourceGroupName string, accountName string, containerName string) error
+	GetAZBlobClient(blobContainerURL string, options *azblob.ClientOptions) (AZBlobClient, error)
+}
+
+type manager struct {
+	cred          azcore.TokenCredential
+	blobContainer armstorage.BlobContainersClient
+}
+
+func NewManager(env env.Interface, subscriptionID string, credential azcore.TokenCredential) (Manager, error) {
+	client, err := armstorage.NewBlobContainersClient(env.Environment(), subscriptionID, credential)
+	if err != nil {
+		return nil, err
+	}
+	return &manager{
+		cred:          credential,
+		blobContainer: client,
+	}, nil
+}
+
+func (m *manager) CreateBlobContainer(ctx context.Context, resourceGroup string, accountName string, containerName string, publicAccess azstorage.PublicAccess) error {
+	needToCreateBlobContainer := false
+
+	_, err := m.blobContainer.Get(
+		ctx,
+		resourceGroup,
+		accountName,
+		containerName,
+		&azstorage.BlobContainersClientGetOptions{},
+	)
+	if err != nil {
+		if !bloberror.HasCode(err, bloberror.ContainerNotFound) {
+			return err
+		}
+		needToCreateBlobContainer = true
+	}
+
+	if !needToCreateBlobContainer {
+		return nil
+	}
+
+	_, err = m.blobContainer.Create(
+		ctx,
+		resourceGroup,
+		accountName,
+		containerName,
+		azstorage.BlobContainer{
+			ContainerProperties: &azstorage.ContainerProperties{
+				PublicAccess: to.Ptr(azstorage.PublicAccessNone),
+			},
+		},
+		&azstorage.BlobContainersClientCreateOptions{},
+	)
+
+	return err
+}
+
+func (m *manager) DeleteBlobContainer(ctx context.Context, resourceGroupName string, accountName string, containerName string) error {
+	_, err := m.blobContainer.Delete(ctx, resourceGroupName, accountName, containerName, &azstorage.BlobContainersClientDeleteOptions{})
+	return err
+}
+
+type AZBlobClient interface {
+	UploadBuffer(ctx context.Context, containerName string, blobName string, buffer []byte, o *blockblob.UploadBufferOptions) (blockblob.UploadBufferResponse, error)
+}
+
+type azBlobClient struct {
+	client *azblob.Client
+}
+
+func (m *manager) GetAZBlobClient(blobContainerURL string, options *azblob.ClientOptions) (AZBlobClient, error) {
+	client, err := azblob.NewClient(blobContainerURL, m.cred, options)
+	if err != nil {
+		return nil, err
+	}
+	return &azBlobClient{client: client}, nil
+}
+
+func (azBlobClient *azBlobClient) UploadBuffer(ctx context.Context, containerName string, blobName string, buffer []byte, o *blockblob.UploadBufferOptions) (blockblob.UploadBufferResponse, error) {
+	return azBlobClient.client.UploadBuffer(ctx, containerName, blobName, buffer, o)
+}

pkg/util/azureclient/azuresdk/armstorage/blobContainers.go

@@ -0,0 +1,41 @@
+package armstorage
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
+
+	"github.com/Azure/ARO-RP/pkg/util/azureclient"
+)
+
+// BlobContainersClient is a minimal interface for Azure BlobContainersClient
+type BlobContainersClient interface {
+	Create(ctx context.Context, resourceGroupName string, accountName string, containerName string, blobContainer armstorage.BlobContainer, options *armstorage.BlobContainersClientCreateOptions) (armstorage.BlobContainersClientCreateResponse, error)
+	Get(ctx context.Context, resourceGroupName string, accountName string, containerName string, options *armstorage.BlobContainersClientGetOptions) (armstorage.BlobContainersClientGetResponse, error)
+	Delete(ctx context.Context, resourceGroupName string, accountName string, containerName string, options *armstorage.BlobContainersClientDeleteOptions) (armstorage.BlobContainersClientDeleteResponse, error)
+}
+
+type blobContainersClient struct {
+	*armstorage.BlobContainersClient
+}
+
+var _ BlobContainersClient = &blobContainersClient{}
+
+// NewBlobContainersClient creates a new BlobContainersClient
+func NewBlobContainersClient(environment *azureclient.AROEnvironment, subscriptionID string, credential azcore.TokenCredential) (BlobContainersClient, error) {
+	options := arm.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Cloud: environment.Cloud,
+		},
+	}
+	clientFactory, err := armstorage.NewClientFactory(subscriptionID, credential, &options)
+	if err != nil {
+		return nil, err
+	}
+	return &blobContainersClient{BlobContainersClient: clientFactory.NewBlobContainersClient()}, nil
+}

pkg/util/azureclient/azuresdk/armstorage/generate.go

@@ -0,0 +1,8 @@
+package armstorage
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate rm -rf ../../../../util/mocks/$GOPACKAGE
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/azuresdk/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/$GOPACKAGE BlobContainersClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/azuresdk/$GOPACKAGE/$GOPACKAGE.go

pkg/util/azureclient/mgmt/storage/blobcontainers.go

@@ -0,0 +1,31 @@
+package storage
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage"
+	"github.com/Azure/go-autorest/autorest"
+
+	"github.com/Azure/ARO-RP/pkg/util/azureclient"
+)
+
+type BlobContainersClient interface {
+	Get(ctx context.Context, resourceGroupName string, accountName string, containerName string) (mgmtstorage.BlobContainer, error)
+	Create(ctx context.Context, resourceGroupName string, accountName string, containerName string, blobContainer mgmtstorage.BlobContainer) (mgmtstorage.BlobContainer, error)
+	Delete(ctx context.Context, resourceGroupName string, accountName string, containerName string) (result autorest.Response, err error)
+}
+
+type blobContainersClient struct {
+	mgmtstorage.BlobContainersClient
+}
+
+func NewBlobContainersClient(environment *azureclient.AROEnvironment, subscriptionID string, authorizer autorest.Authorizer) *blobContainersClient {
+	client := mgmtstorage.NewBlobContainersClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID)
+	client.Authorizer = authorizer
+	return &blobContainersClient{
+		BlobContainersClient: client,
+	}
+}