Skip to content

Commit

Permalink
Add e2e tests for multiple ips per loadbalancer (Azure#3183)
Browse files Browse the repository at this point in the history
* add azureclient for 2023-07-01-preview
* add UpdateAndWait to oc azure client
* add e2e test for managed outbound IPs
  • Loading branch information
tony-schndr authored and ventifus committed Feb 7, 2024
1 parent 447445d commit d5937fa
Show file tree
Hide file tree
Showing 6 changed files with 323 additions and 4 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package redhatopenshift

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

import (
"context"
"crypto/tls"
"net/http"
"time"

"github.com/Azure/go-autorest/autorest"

mgmtredhatopenshift20230701preview "github.com/Azure/ARO-RP/pkg/client/services/redhatopenshift/mgmt/2023-07-01-preview/redhatopenshift"
"github.com/Azure/ARO-RP/pkg/env"
"github.com/Azure/ARO-RP/pkg/util/azureclient"
)

// OpenShiftClustersClient is a minimal interface for azure OpenshiftClustersClient
type OpenShiftClustersClient interface {
ListCredentials(ctx context.Context, resourceGroupName string, resourceName string) (result mgmtredhatopenshift20230701preview.OpenShiftClusterCredentials, err error)
Get(ctx context.Context, resourceGroupName string, resourceName string) (result mgmtredhatopenshift20230701preview.OpenShiftCluster, err error)
OpenShiftClustersClientAddons
}

type openShiftClustersClient struct {
mgmtredhatopenshift20230701preview.OpenShiftClustersClient
}

var _ OpenShiftClustersClient = &openShiftClustersClient{}

// NewOpenShiftClustersClient creates a new OpenShiftClustersClient
func NewOpenShiftClustersClient(environment *azureclient.AROEnvironment, subscriptionID string, authorizer autorest.Authorizer) OpenShiftClustersClient {
var client mgmtredhatopenshift20230701preview.OpenShiftClustersClient
if env.IsLocalDevelopmentMode() {
client = mgmtredhatopenshift20230701preview.NewOpenShiftClustersClientWithBaseURI("https://localhost:8443", subscriptionID)
client.Sender = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // #nosec G402
},
},
}
} else {
client = mgmtredhatopenshift20230701preview.NewOpenShiftClustersClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID)
client.Authorizer = authorizer
}
client.PollingDelay = 10 * time.Second
client.PollingDuration = 2 * time.Hour

return &openShiftClustersClient{
OpenShiftClustersClient: client,
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package redhatopenshift

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

import (
"context"

mgmtredhatopenshift20230701preview "github.com/Azure/ARO-RP/pkg/client/services/redhatopenshift/mgmt/2023-07-01-preview/redhatopenshift"
)

// OpenShiftClustersClientAddons contains addons for OpenShiftClustersClient
type OpenShiftClustersClientAddons interface {
CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, resourceName string, parameters mgmtredhatopenshift20230701preview.OpenShiftCluster) error
UpdateAndWait(ctx context.Context, resourceGroupName string, resourceName string, parameters mgmtredhatopenshift20230701preview.OpenShiftClusterUpdate) error
DeleteAndWait(ctx context.Context, resourceGroupName string, resourceName string) error
List(ctx context.Context) (clusters []mgmtredhatopenshift20230701preview.OpenShiftCluster, err error)
ListByResourceGroup(ctx context.Context, resourceGroupName string) (clusters []mgmtredhatopenshift20230701preview.OpenShiftCluster, err error)
}

func (c *openShiftClustersClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, resourceName string, parameters mgmtredhatopenshift20230701preview.OpenShiftCluster) error {
future, err := c.CreateOrUpdate(ctx, resourceGroupName, resourceName, parameters)
if err != nil {
return err
}

return future.WaitForCompletionRef(ctx, c.Client)
}

func (c *openShiftClustersClient) UpdateAndWait(ctx context.Context, resourceGroupName string, resourceName string, parameters mgmtredhatopenshift20230701preview.OpenShiftClusterUpdate) error {
future, err := c.Update(ctx, resourceGroupName, resourceName, parameters)
if err != nil {
return err
}

return future.WaitForCompletionRef(ctx, c.Client)
}

func (c *openShiftClustersClient) DeleteAndWait(ctx context.Context, resourceGroupName string, resourceName string) error {
future, err := c.Delete(ctx, resourceGroupName, resourceName)
if err != nil {
return err
}

return future.WaitForCompletionRef(ctx, c.Client)
}

func (c *openShiftClustersClient) List(ctx context.Context) (clusters []mgmtredhatopenshift20230701preview.OpenShiftCluster, err error) {
page, err := c.OpenShiftClustersClient.List(ctx)
if err != nil {
return nil, err
}

for page.NotDone() {
clusters = append(clusters, page.Values()...)

err = page.NextWithContext(ctx)
if err != nil {
return nil, err
}
}

return clusters, nil
}

func (c *openShiftClustersClient) ListByResourceGroup(ctx context.Context, resourceGroupName string) (clusters []mgmtredhatopenshift20230701preview.OpenShiftCluster, err error) {
page, err := c.OpenShiftClustersClient.ListByResourceGroup(ctx, resourceGroupName)
if err != nil {
return nil, err
}

for page.NotDone() {
clusters = append(clusters, page.Values()...)

err = page.NextWithContext(ctx)
if err != nil {
return nil, err
}
}

return clusters, nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package redhatopenshift

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

import (
"crypto/tls"
"net/http"

"github.com/Azure/go-autorest/autorest"

mgmtredhatopenshift20230701preview "github.com/Azure/ARO-RP/pkg/client/services/redhatopenshift/mgmt/2023-07-01-preview/redhatopenshift"
"github.com/Azure/ARO-RP/pkg/env"
"github.com/Azure/ARO-RP/pkg/util/azureclient"
)

// OperationsClient is a minimal interface for azure OperationsClient
type OperationsClient interface {
OperationsClientAddons
}

type operationsClient struct {
mgmtredhatopenshift20230701preview.OperationsClient
}

var _ OperationsClient = &operationsClient{}

// NewOperationsClient creates a new OperationsClient
func NewOperationsClient(environment *azureclient.AROEnvironment, subscriptionID string, authorizer autorest.Authorizer) OperationsClient {
var client mgmtredhatopenshift20230701preview.OperationsClient
if env.IsLocalDevelopmentMode() {
client = mgmtredhatopenshift20230701preview.NewOperationsClientWithBaseURI("https://localhost:8443", subscriptionID)
client.Sender = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, // #nosec G402
},
},
}
} else {
client = mgmtredhatopenshift20230701preview.NewOperationsClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID)
client.Authorizer = authorizer
}

return &operationsClient{
OperationsClient: client,
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package redhatopenshift

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

import (
"context"

mgmtredhatopenshift20230701preview "github.com/Azure/ARO-RP/pkg/client/services/redhatopenshift/mgmt/2023-07-01-preview/redhatopenshift"
)

// OperationsClientAddons contains addons for OperationsClient
type OperationsClientAddons interface {
List(ctx context.Context) (operations []mgmtredhatopenshift20230701preview.Operation, err error)
}

func (c *operationsClient) List(ctx context.Context) (operations []mgmtredhatopenshift20230701preview.Operation, err error) {
page, err := c.OperationsClient.List(ctx)
if err != nil {
return nil, err
}

for page.NotDone() {
operations = append(operations, page.Values()...)

err = page.NextWithContext(ctx)
if err != nil {
return nil, err
}
}

return operations, nil
}
13 changes: 9 additions & 4 deletions test/e2e/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
redhatopenshift20220904 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2022-09-04/redhatopenshift"
redhatopenshift20230701preview "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2023-07-01-preview/redhatopenshift"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage"
"github.com/Azure/ARO-RP/pkg/util/cluster"
utillog "github.com/Azure/ARO-RP/pkg/util/log"
Expand All @@ -54,8 +55,9 @@ var (
)

type clientSet struct {
Operations redhatopenshift20220904.OperationsClient
OpenshiftClusters redhatopenshift20220904.OpenShiftClustersClient
Operations redhatopenshift20220904.OperationsClient
OpenshiftClusters redhatopenshift20220904.OpenShiftClustersClient
OpenshiftClustersPreview redhatopenshift20230701preview.OpenShiftClustersClient

VirtualMachines compute.VirtualMachinesClient
Resources features.ResourcesClient
Expand All @@ -65,6 +67,7 @@ type clientSet struct {
NetworkSecurityGroups network.SecurityGroupsClient
Subnet network.SubnetsClient
Storage storage.AccountsClient
LoadBalancers network.LoadBalancersClient

RestConfig *rest.Config
HiveRestConfig *rest.Config
Expand Down Expand Up @@ -336,8 +339,9 @@ func newClientSet(ctx context.Context) (*clientSet, error) {
}

return &clientSet{
Operations: redhatopenshift20220904.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
OpenshiftClusters: redhatopenshift20220904.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
Operations: redhatopenshift20220904.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
OpenshiftClusters: redhatopenshift20220904.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),
OpenshiftClustersPreview: redhatopenshift20230701preview.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer),

VirtualMachines: compute.NewVirtualMachinesClient(_env.Environment(), _env.SubscriptionID(), authorizer),
Resources: features.NewResourcesClient(_env.Environment(), _env.SubscriptionID(), authorizer),
Expand All @@ -347,6 +351,7 @@ func newClientSet(ctx context.Context) (*clientSet, error) {
Subnet: network.NewSubnetsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
NetworkSecurityGroups: network.NewSecurityGroupsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
Storage: storage.NewAccountsClient(_env.Environment(), _env.SubscriptionID(), authorizer),
LoadBalancers: network.NewLoadBalancersClient(_env.Environment(), _env.SubscriptionID(), authorizer),

RestConfig: restconfig,
HiveRestConfig: hiveRestConfig,
Expand Down
97 changes: 97 additions & 0 deletions test/e2e/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
"github.com/Azure/go-autorest/autorest/to"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

mgmtredhatopenshift20230701preview "github.com/Azure/ARO-RP/pkg/client/services/redhatopenshift/mgmt/2023-07-01-preview/redhatopenshift"
"github.com/Azure/ARO-RP/pkg/util/stringutils"
)

var _ = Describe("Update clusters", func() {
Expand Down Expand Up @@ -41,3 +48,93 @@ var _ = Describe("Update clusters", func() {
Expect(oc.Tags).To(HaveKeyWithValue("key", &value))
})
})

var _ = Describe("Update cluster Managed Outbound IPs", func() {
var lbName string
var rgName string

var _ = BeforeEach(func(ctx context.Context) {
By("ensuring the public loadbalancer starts with one outbound IP")
oc, err := clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName)
Expect(err).NotTo(HaveOccurred())

lbName, err = getPublicLoadBalancerName(ctx)
Expect(err).NotTo(HaveOccurred())

rgName = stringutils.LastTokenByte(*oc.ClusterProfile.ResourceGroupID, '/')
lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "")
Expect(err).NotTo(HaveOccurred())

if getOutboundIPsCount(lb) != 1 {
By("sending the PATCH request to set ManagedOutboundIPs.Count to 1")
err = clients.OpenshiftClustersPreview.UpdateAndWait(ctx, vnetResourceGroup, clusterName, newManagedOutboundIPUpdateBody(1))
Expect(err).NotTo(HaveOccurred())
}
})

It("must be possible to increase and decrease IP Addresses on the public loadbalancer", func(ctx context.Context) {
By("sending the PATCH request to increase Managed Outbound IPs")
err := clients.OpenshiftClustersPreview.UpdateAndWait(ctx, vnetResourceGroup, clusterName, newManagedOutboundIPUpdateBody(5))
Expect(err).NotTo(HaveOccurred())

By("getting the cluster resource")
oc, err := clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName)
Expect(err).NotTo(HaveOccurred())

By("checking effectiveOutboundIPs has been updated")
Expect(len(*oc.NetworkProfile.LoadBalancerProfile.EffectiveOutboundIps)).To(Equal(5))

By("checking outbound-rule-4 has required number IPs")
lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "")
Expect(err).NotTo(HaveOccurred())
Expect(getOutboundIPsCount(lb)).To(Equal(5))

By("sending the PATCH request to decrease Managed Outbound IPs")
err = clients.OpenshiftClustersPreview.UpdateAndWait(ctx, vnetResourceGroup, clusterName, newManagedOutboundIPUpdateBody(1))
Expect(err).NotTo(HaveOccurred())

By("getting the cluster resource")
oc, err = clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName)
Expect(err).NotTo(HaveOccurred())

By("checking effectiveOutboundIPs has been updated")
Expect(len(*oc.NetworkProfile.LoadBalancerProfile.EffectiveOutboundIps)).To(Equal(1))

By("checking outbound-rule-4 has required number of IPs")
lb, err = clients.LoadBalancers.Get(ctx, rgName, lbName, "")
Expect(err).NotTo(HaveOccurred())
Expect(getOutboundIPsCount(lb)).To(Equal(1))
})
})

func getPublicLoadBalancerName(ctx context.Context) (string, error) {
co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{})
if err != nil {
return "", err
}
return co.Spec.InfraID, err
}

func newManagedOutboundIPUpdateBody(managedOutboundIPCount int32) mgmtredhatopenshift20230701preview.OpenShiftClusterUpdate {
return mgmtredhatopenshift20230701preview.OpenShiftClusterUpdate{
OpenShiftClusterProperties: &mgmtredhatopenshift20230701preview.OpenShiftClusterProperties{
NetworkProfile: &mgmtredhatopenshift20230701preview.NetworkProfile{
LoadBalancerProfile: &mgmtredhatopenshift20230701preview.LoadBalancerProfile{
ManagedOutboundIps: &mgmtredhatopenshift20230701preview.ManagedOutboundIPs{
Count: to.Int32Ptr(managedOutboundIPCount),
},
},
},
},
}
}

func getOutboundIPsCount(lb mgmtnetwork.LoadBalancer) int {
numOfIPs := 0
for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules {
if *obRule.Name == "outbound-rule-v4" {
numOfIPs = len(*obRule.FrontendIPConfigurations)
}
}
return numOfIPs
}

0 comments on commit d5937fa

Please sign in to comment.