From 84816de0e43d611efdfbfa1aef1bf7f40f4bedf8 Mon Sep 17 00:00:00 2001 From: Ayato Tokubi Date: Fri, 27 Sep 2024 18:44:06 +0000 Subject: [PATCH] SDK2: Remove deprecated network sdk from e2e. (#3859) * subnets * remove deprecated security group usage from e2e setup * remove deprecated interface sdk from e2e. * remove deprecated loadbalancer sdk from e2e. * remove deprecated virtual networks sdk from e2e. * remove deprecated subnets sdk from e2e --- pkg/api/util/subnet/subnet.go | 3 +- pkg/monitor/azure/nsg/nsg.go | 9 ++- .../azuresdk/armnetwork/subnets.go | 13 ++-- .../azuresdk/armnetwork/subnets_addons.go | 34 ++++++++++ .../azuresdk/armnetwork/virtualnetworks.go | 29 +++++++++ .../azuresdk/armnetwork/armnetwork.go | 28 +++++++++ test/e2e/adminapi_delete_managedresource.go | 12 ++-- test/e2e/adminapi_resources.go | 9 ++- test/e2e/cluster.go | 34 +++++----- test/e2e/dns.go | 28 +++++---- test/e2e/operator.go | 48 +++++++------- test/e2e/setup.go | 63 +++++++++++++++---- test/e2e/update.go | 20 +++--- 13 files changed, 238 insertions(+), 92 deletions(-) create mode 100644 pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go create mode 100644 pkg/util/azureclient/azuresdk/armnetwork/virtualnetworks.go diff --git a/pkg/api/util/subnet/subnet.go b/pkg/api/util/subnet/subnet.go index be58f8ad6ee..20158f8b02f 100644 --- a/pkg/api/util/subnet/subnet.go +++ b/pkg/api/util/subnet/subnet.go @@ -25,8 +25,7 @@ func Split(subnetID string) (string, string, error) { return strings.Join(parts[:len(parts)-2], "/"), parts[len(parts)-1], nil } -// NetworkSecurityGroupID returns the NetworkSecurityGroup ID for a given subnet -// ID +// NetworkSecurityGroupID returns the NetworkSecurityGroup ID for a given subnet ID func NetworkSecurityGroupID(oc *api.OpenShiftCluster, subnetID string) (string, error) { infraID := oc.Properties.InfraID if infraID == "" { diff --git a/pkg/monitor/azure/nsg/nsg.go b/pkg/monitor/azure/nsg/nsg.go index 8054594f830..93af0be5817 100644 --- a/pkg/monitor/azure/nsg/nsg.go +++ b/pkg/monitor/azure/nsg/nsg.go @@ -75,10 +75,13 @@ func NewMonitor(log *logrus.Entry, oc *api.OpenShiftCluster, e env.Interface, su return &monitoring.NoOpMonitor{Wg: wg} } - options := arm.ClientOptions{ - ClientOptions: e.Environment().ClientCertificateCredentialOptions().ClientOptions, + clientOptions := arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: e.Environment().Cloud, + }, } - client, err := armnetwork.NewSubnetsClient(subscriptionID, token, &options) + + client, err := sdknetwork.NewSubnetsClient(subscriptionID, token, &clientOptions) if err != nil { log.Error("Unable to create the subnet client for NSG monitoring", err) emitter.EmitGauge(MetricFailedNSGMonitorCreation, int64(1), dims) diff --git a/pkg/util/azureclient/azuresdk/armnetwork/subnets.go b/pkg/util/azureclient/azuresdk/armnetwork/subnets.go index 10ec38675c5..2bc148abf6f 100644 --- a/pkg/util/azureclient/azuresdk/armnetwork/subnets.go +++ b/pkg/util/azureclient/azuresdk/armnetwork/subnets.go @@ -7,24 +7,23 @@ import ( "context" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - sdknetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azcore" ) // SubnetsClient is a minimal interface for azure-sdk-for-go subnets client type SubnetsClient interface { - Get(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *sdknetwork.SubnetsClientGetOptions) (sdknetwork.SubnetsClientGetResponse, error) + Get(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientGetOptions) (armnetwork.SubnetsClientGetResponse, error) + SubnetsClientAddons } type subnetsClient struct { - *sdknetwork.SubnetsClient + *armnetwork.SubnetsClient } -var _ SubnetsClient = (*subnetsClient)(nil) - func NewSubnetsClient(subscriptionID string, credential azcore.TokenCredential, options *arm.ClientOptions) (SubnetsClient, error) { - client, err := sdknetwork.NewSubnetsClient(subscriptionID, credential, options) + client, err := armnetwork.NewSubnetsClient(subscriptionID, credential, options) - return subnetsClient{client}, err + return &subnetsClient{client}, err } diff --git a/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go b/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go new file mode 100644 index 00000000000..57fe37407c5 --- /dev/null +++ b/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go @@ -0,0 +1,34 @@ +package armnetwork + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" +) + +// SubnetsClientAddons contains addons for SubnetsClient +type SubnetsClientAddons interface { + CreateOrUpdateAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, subnetParameters armnetwork.Subnet, options *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) (err error) + DeleteAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientBeginDeleteOptions) error +} + +func (c *subnetsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, subnetParameters armnetwork.Subnet, options *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) error { + poller, err := c.SubnetsClient.BeginCreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, subnetName, subnetParameters, options) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + return err +} + +func (c *subnetsClient) DeleteAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientBeginDeleteOptions) error { + poller, err := c.SubnetsClient.BeginDelete(ctx, resourceGroupName, virtualNetworkName, subnetName, options) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + return err +} diff --git a/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworks.go b/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworks.go new file mode 100644 index 00000000000..0193efb73f1 --- /dev/null +++ b/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworks.go @@ -0,0 +1,29 @@ +package armnetwork + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" + + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azcore" +) + +// VirtualNetworksClient is a minimal interface for azure VirtualNetworksClient +type VirtualNetworksClient interface { + Get(ctx context.Context, resourceGroupName string, virtualNetworkName string, options *armnetwork.VirtualNetworksClientGetOptions) (vnet armnetwork.VirtualNetworksClientGetResponse, err error) +} + +type virtualNetworksClient struct { + *armnetwork.VirtualNetworksClient +} + +// NewVirtualNetworksClient creates a new VirtualNetworksClient +func NewVirtualNetworksClient(subscriptionID string, credential azcore.TokenCredential, options *arm.ClientOptions) (VirtualNetworksClient, error) { + client, err := armnetwork.NewVirtualNetworksClient(subscriptionID, credential, options) + + return &virtualNetworksClient{client}, err +} diff --git a/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go b/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go index 9ab17f4a9fa..d3e41306636 100644 --- a/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go +++ b/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go @@ -506,6 +506,34 @@ func (m *MockSubnetsClient) EXPECT() *MockSubnetsClientMockRecorder { return m.recorder } +// CreateOrUpdateAndWait mocks base method. +func (m *MockSubnetsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2, arg3 string, arg4 armnetwork.Subnet, arg5 *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait. +func (mr *MockSubnetsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockSubnetsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// DeleteAndWait mocks base method. +func (m *MockSubnetsClient) DeleteAndWait(arg0 context.Context, arg1, arg2, arg3 string, arg4 *armnetwork.SubnetsClientBeginDeleteOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAndWait indicates an expected call of DeleteAndWait. +func (mr *MockSubnetsClientMockRecorder) DeleteAndWait(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockSubnetsClient)(nil).DeleteAndWait), arg0, arg1, arg2, arg3, arg4) +} + // Get mocks base method. func (m *MockSubnetsClient) Get(arg0 context.Context, arg1, arg2, arg3 string, arg4 *armnetwork.SubnetsClientGetOptions) (armnetwork.SubnetsClientGetResponse, error) { m.ctrl.T.Helper() diff --git a/test/e2e/adminapi_delete_managedresource.go b/test/e2e/adminapi_delete_managedresource.go index 323c9409b2b..f687f89852e 100644 --- a/test/e2e/adminapi_delete_managedresource.go +++ b/test/e2e/adminapi_delete_managedresource.go @@ -72,14 +72,16 @@ var _ = Describe("[Admin API] Delete managed resource action", func() { lbName, err := getInfraID(ctx) Expect(err).NotTo(HaveOccurred()) - lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "") + lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, nil) Expect(err).NotTo(HaveOccurred()) - for _, fipConfig := range *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations { - if !strings.Contains(*fipConfig.PublicIPAddress.ID, "default-v4") && !strings.Contains(*fipConfig.PublicIPAddress.ID, "pip-v4") { - lbRuleID = *(*fipConfig.LoadBalancingRules)[0].ID + for _, fipConfig := range lb.Properties.FrontendIPConfigurations { + Expect(fipConfig.Properties.PublicIPAddress).NotTo(BeNil()) + if !strings.Contains(*fipConfig.Properties.PublicIPAddress.ID, "default-v4") && !strings.Contains(*fipConfig.Properties.PublicIPAddress.ID, "pip-v4") { + Expect(fipConfig.Properties.LoadBalancingRules).To(HaveLen(1)) + lbRuleID = *fipConfig.Properties.LoadBalancingRules[0].ID fipConfigID = *fipConfig.ID - pipAddressID = *fipConfig.PublicIPAddress.ID + pipAddressID = *fipConfig.Properties.PublicIPAddress.ID } } diff --git a/test/e2e/adminapi_resources.go b/test/e2e/adminapi_resources.go index f7be4e1ee78..881db1c0c7d 100644 --- a/test/e2e/adminapi_resources.go +++ b/test/e2e/adminapi_resources.go @@ -59,17 +59,16 @@ var _ = Describe("[Admin API] List Azure resources action", func() { subnets[strings.ToLower(*p.SubnetID)] = struct{}{} } - vnet, err := clients.VirtualNetworks.Get(ctx, r.ResourceGroup, r.ResourceName, "") + vnet, err := clients.VirtualNetworks.Get(ctx, r.ResourceGroup, r.ResourceName, nil) Expect(err).NotTo(HaveOccurred()) - for _, subnet := range *vnet.Subnets { + for _, subnet := range vnet.Properties.Subnets { if _, ok := subnets[strings.ToLower(*subnet.ID)]; !ok { continue } - if subnet.SubnetPropertiesFormat != nil && - subnet.RouteTable != nil { - expectedResourceIDs = append(expectedResourceIDs, strings.ToLower(*subnet.RouteTable.ID)) + if subnet.Properties != nil && subnet.Properties.RouteTable != nil { + expectedResourceIDs = append(expectedResourceIDs, strings.ToLower(*subnet.Properties.RouteTable.ID)) } } diff --git a/test/e2e/cluster.go b/test/e2e/cluster.go index 4c8c8c3de83..32c82188036 100644 --- a/test/e2e/cluster.go +++ b/test/e2e/cluster.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-09-01/storage" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" @@ -94,20 +94,21 @@ var _ = Describe("Cluster", Serial, func() { vnetR, err := azure.ParseResourceID(vnetID) Expect(err).NotTo(HaveOccurred()) - mgmtSubnet, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, "") + resp, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, nil) Expect(err).NotTo(HaveOccurred()) + subnet := resp.Subnet - if mgmtSubnet.SubnetPropertiesFormat == nil { - mgmtSubnet.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{} + if subnet.Properties == nil { + subnet.Properties = &armnetwork.SubnetPropertiesFormat{} } - if mgmtSubnet.SubnetPropertiesFormat.ServiceEndpoints == nil { - mgmtSubnet.SubnetPropertiesFormat.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{} + if subnet.Properties.ServiceEndpoints == nil { + subnet.Properties.ServiceEndpoints = []*armnetwork.ServiceEndpointPropertiesFormat{} } // Check whether service endpoint is already there before trying to add // it; trying to add a duplicate results in an error - for _, se := range *mgmtSubnet.ServiceEndpoints { + for _, se := range subnet.Properties.ServiceEndpoints { if se.Service != nil && *se.Service == "Microsoft.Storage" { subnetAlreadyHasStorageEndpoint = true break @@ -115,14 +116,14 @@ var _ = Describe("Cluster", Serial, func() { } if !subnetAlreadyHasStorageEndpoint { - storageEndpoint := mgmtnetwork.ServiceEndpointPropertiesFormat{ + storageEndpoint := armnetwork.ServiceEndpointPropertiesFormat{ Service: to.StringPtr("Microsoft.Storage"), - Locations: &[]string{"*"}, + Locations: []*string{to.StringPtr("*")}, } - *mgmtSubnet.ServiceEndpoints = append(*mgmtSubnet.ServiceEndpoints, storageEndpoint) + subnet.Properties.ServiceEndpoints = append(subnet.Properties.ServiceEndpoints, &storageEndpoint) - err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, mgmtSubnet) + err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, subnet, nil) Expect(err).NotTo(HaveOccurred()) } } @@ -193,16 +194,17 @@ var _ = Describe("Cluster", Serial, func() { vnetR, err := azure.ParseResourceID(vnetID) Expect(err).NotTo(HaveOccurred()) - mgmtSubnet, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, "") + resp, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, nil) Expect(err).NotTo(HaveOccurred()) + subnet := resp.Subnet - if mgmtSubnet.SubnetPropertiesFormat == nil { - mgmtSubnet.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{} + if subnet.Properties == nil { + subnet.Properties = &armnetwork.SubnetPropertiesFormat{} } - mgmtSubnet.SubnetPropertiesFormat.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{} + subnet.Properties.ServiceEndpoints = []*armnetwork.ServiceEndpointPropertiesFormat{} - err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, mgmtSubnet) + err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, subnet, nil) Expect(err).NotTo(HaveOccurred()) } } diff --git a/test/e2e/dns.go b/test/e2e/dns.go index a57a7d3aa40..4c93156c945 100644 --- a/test/e2e/dns.go +++ b/test/e2e/dns.go @@ -20,7 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork" "github.com/Azure/ARO-RP/pkg/util/ready" "github.com/Azure/ARO-RP/pkg/util/stringutils" ) @@ -77,15 +77,15 @@ var _ = Describe("ARO cluster DNS", func() { clusterResourceGroup := stringutils.LastTokenByte(*oc.OpenShiftClusterProperties.ClusterProfile.ResourceGroupID, '/') for wn := range workerNodes { - nic, err := clients.Interfaces.Get(ctx, clusterResourceGroup, nicName(wn), "") + resp, err := clients.Interfaces.Get(ctx, clusterResourceGroup, nicName(wn), nil) Expect(err).NotTo(HaveOccurred()) + nic := resp.Interface - Expect(nic.InterfacePropertiesFormat).NotTo(BeNil()) - Expect(nic.IPConfigurations).NotTo(BeNil()) - Expect(*nic.IPConfigurations).To(HaveLen(1)) - Expect((*nic.IPConfigurations)[0].InterfaceIPConfigurationPropertiesFormat).NotTo(BeNil()) - Expect((*nic.IPConfigurations)[0].PrivateIPAddress).NotTo(BeNil()) - workerNodes[wn] = *(*nic.IPConfigurations)[0].PrivateIPAddress + Expect(nic.Properties).NotTo(BeNil()) + Expect(nic.Properties.IPConfigurations).To(HaveLen(1)) + Expect(nic.Properties.IPConfigurations[0].Properties).NotTo(BeNil()) + Expect(nic.Properties.IPConfigurations[0].Properties.PrivateIPAddress).NotTo(BeNil()) + workerNodes[wn] = *nic.Properties.IPConfigurations[0].Properties.PrivateIPAddress } By("preparing to read resolv.conf from each of the worker nodes by allowing the test namespace's ServiceAccount to use the hostmount-anyuid SecurityContextConstraint") @@ -380,14 +380,18 @@ func verifyResolvConf( return nil } -func toggleAcceleratedNetworking(ctx context.Context, interfaces network.InterfacesClient, clusterResourceGroup string, nodeName string, enabled bool) error { - nic, err := interfaces.Get(ctx, clusterResourceGroup, nicName(nodeName), "") +func toggleAcceleratedNetworking(ctx context.Context, interfaces armnetwork.InterfacesClient, clusterResourceGroup string, nodeName string, enabled bool) error { + resp, err := interfaces.Get(ctx, clusterResourceGroup, nicName(nodeName), nil) if err != nil { return err } + nic := resp.Interface - nic.EnableAcceleratedNetworking = to.BoolPtr(enabled) - err = clients.Interfaces.CreateOrUpdateAndWait(ctx, clusterResourceGroup, nicName(nodeName), nic) + if nic.Properties == nil { + return fmt.Errorf("NIC properties are nil") + } + nic.Properties.EnableAcceleratedNetworking = to.BoolPtr(enabled) + err = clients.Interfaces.CreateOrUpdateAndWait(ctx, clusterResourceGroup, nicName(nodeName), nic, nil) return err } diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 5af2c16e41e..097ac9d149d 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" configv1 "github.com/openshift/api/config/v1" @@ -312,7 +312,7 @@ func subnetReconciliationAnnotationExists(annotations map[string]string) bool { var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { var vnetName, location, resourceGroup string var subnetsToReconcile map[string]*string - var testNSG mgmtnetwork.SecurityGroup + var testNSG armnetwork.SecurityGroup const nsg = "e2e-nsg" const emptyMachineSet = "e2e-test-machineset" @@ -344,23 +344,24 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { // This is expensive but will prevent flakes. By("gathering existing subnet NSGs") for subnet := range subnetsToReconcile { - subnetObject, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, "") + subnetObject, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, nil) Expect(err).NotTo(HaveOccurred()) - subnetsToReconcile[subnet] = subnetObject.NetworkSecurityGroup.ID + subnetsToReconcile[subnet] = subnetObject.Properties.NetworkSecurityGroup.ID } } cleanUpSubnetNSGs := func(ctx context.Context) { By("cleaning up subnet NSGs") for subnet := range subnetsToReconcile { - subnetObject, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, "") + resp, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, nil) Expect(err).NotTo(HaveOccurred()) + subnetObject := resp.Subnet - if subnetObject.NetworkSecurityGroup.ID != subnetsToReconcile[subnet] { - subnetObject.NetworkSecurityGroup.ID = subnetsToReconcile[subnet] + if subnetObject.Properties.NetworkSecurityGroup.ID != subnetsToReconcile[subnet] { + subnetObject.Properties.NetworkSecurityGroup.ID = subnetsToReconcile[subnet] - err = clients.Subnet.CreateOrUpdateAndWait(ctx, resourceGroup, vnetName, subnet, subnetObject) + err = clients.Subnet.CreateOrUpdateAndWait(ctx, resourceGroup, vnetName, subnet, subnetObject, nil) Expect(err).NotTo(HaveOccurred()) } } @@ -368,17 +369,18 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { createE2ENSG := func(ctx context.Context) { By("creating an empty test NSG") - testNSG = mgmtnetwork.SecurityGroup{ - Location: &location, - Name: to.StringPtr(nsg), - Type: to.StringPtr("Microsoft.Network/networkSecurityGroups"), - SecurityGroupPropertiesFormat: &mgmtnetwork.SecurityGroupPropertiesFormat{}, + testNSG = armnetwork.SecurityGroup{ + Location: &location, + Name: to.StringPtr(nsg), + Type: to.StringPtr("Microsoft.Network/networkSecurityGroups"), + Properties: &armnetwork.SecurityGroupPropertiesFormat{}, } - err := clients.NetworkSecurityGroups.CreateOrUpdateAndWait(ctx, resourceGroup, nsg, testNSG) + err := clients.NetworkSecurityGroups.CreateOrUpdateAndWait(ctx, resourceGroup, nsg, testNSG, nil) Expect(err).NotTo(HaveOccurred()) By("getting the freshly created test NSG resource") - testNSG, err = clients.NetworkSecurityGroups.Get(ctx, resourceGroup, nsg, "") + resp, err := clients.NetworkSecurityGroups.Get(ctx, resourceGroup, nsg, nil) + testNSG = resp.SecurityGroup Expect(err).NotTo(HaveOccurred()) } @@ -399,7 +401,7 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { cleanUpSubnetNSGs(ctx) By("deleting test NSG") - err := clients.NetworkSecurityGroups.DeleteAndWait(ctx, resourceGroup, nsg) + err := clients.NetworkSecurityGroups.DeleteAndWait(ctx, resourceGroup, nsg, nil) if err != nil { log.Warn(err) } @@ -411,16 +413,20 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { MatchError(kerrors.IsNotFound), )) }) + It("must reconcile list of subnets when NSG is changed", func(ctx context.Context) { for subnet := range subnetsToReconcile { By(fmt.Sprintf("assigning test NSG to subnet %q", subnet)) // Gets current subnet NSG and then updates it to testNSG. - subnetObject, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, "") + resp, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, nil) Expect(err).NotTo(HaveOccurred()) + subnetObject := resp.Subnet - subnetObject.NetworkSecurityGroup = &testNSG + subnetObject.Properties.NetworkSecurityGroup = &armnetwork.SecurityGroup{ + ID: testNSG.ID, + } - err = clients.Subnet.CreateOrUpdateAndWait(ctx, resourceGroup, vnetName, subnet, subnetObject) + err = clients.Subnet.CreateOrUpdateAndWait(ctx, resourceGroup, vnetName, subnet, subnetObject, nil) Expect(err).NotTo(HaveOccurred()) } @@ -448,9 +454,9 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { for subnet, correctNSG := range subnetsToReconcile { By(fmt.Sprintf("waiting for the subnet %q to be reconciled so it includes the original cluster NSG", subnet)) Eventually(func(g Gomega, ctx context.Context) { - s, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, "") + s, err := clients.Subnet.Get(ctx, resourceGroup, vnetName, subnet, nil) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(*s.NetworkSecurityGroup.ID).To(Equal(*correctNSG)) + g.Expect(*s.Properties.NetworkSecurityGroup.ID).To(Equal(*correctNSG)) co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) g.Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/setup.go b/test/e2e/setup.go index 52ee6952a1e..fd606bffb9f 100644 --- a/test/e2e/setup.go +++ b/test/e2e/setup.go @@ -8,6 +8,7 @@ import ( "embed" "fmt" "math" + "net/http" "net/url" "os" "os/exec" @@ -18,6 +19,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/davecgh/go-spew/spew" "github.com/jongio/azidext/go/azidext" @@ -41,9 +44,11 @@ import ( "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/hive" aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" + "github.com/Azure/ARO-RP/pkg/util/azureclient" + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork" + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/common" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" - "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" redhatopenshift20231122 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2023-11-22/redhatopenshift" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage" "github.com/Azure/ARO-RP/pkg/util/cluster" @@ -73,14 +78,14 @@ type clientSet struct { VirtualMachines compute.VirtualMachinesClient Resources features.ResourcesClient - VirtualNetworks network.VirtualNetworksClient DiskEncryptionSets compute.DiskEncryptionSetsClient Disks compute.DisksClient - NetworkSecurityGroups network.SecurityGroupsClient - Subnet network.SubnetsClient - Interfaces network.InterfacesClient + Interfaces armnetwork.InterfacesClient + LoadBalancers armnetwork.LoadBalancersClient + NetworkSecurityGroups armnetwork.SecurityGroupsClient + Subnet armnetwork.SubnetsClient + VirtualNetworks armnetwork.VirtualNetworksClient Storage storage.AccountsClient - LoadBalancers network.LoadBalancersClient Dynamic dynamic.Client RestConfig *rest.Config @@ -375,20 +380,56 @@ func newClientSet(ctx context.Context) (*clientSet, error) { } } + customRoundTripper := azureclient.NewCustomRoundTripper(http.DefaultTransport) + clientOptions := &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: _env.Environment().Cloud, + Retry: common.RetryOptions, + Transport: &http.Client{ + Transport: customRoundTripper, + }, + }, + } + + interfacesClient, err := armnetwork.NewInterfacesClient(_env.SubscriptionID(), tokenCredential, clientOptions) + if err != nil { + return nil, err + } + + loadBalancersClient, err := armnetwork.NewLoadBalancersClient(_env.SubscriptionID(), tokenCredential, clientOptions) + if err != nil { + return nil, err + } + + securityGroupsClient, err := armnetwork.NewSecurityGroupsClient(_env.SubscriptionID(), tokenCredential, clientOptions) + if err != nil { + return nil, err + } + + subnetsClient, err := armnetwork.NewSubnetsClient(_env.SubscriptionID(), tokenCredential, clientOptions) + if err != nil { + return nil, err + } + + virtualNetworksClient, err := armnetwork.NewVirtualNetworksClient(_env.SubscriptionID(), tokenCredential, clientOptions) + if err != nil { + return nil, err + } + return &clientSet{ Operations: redhatopenshift20231122.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer), OpenshiftClusters: redhatopenshift20231122.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer), VirtualMachines: compute.NewVirtualMachinesClient(_env.Environment(), _env.SubscriptionID(), authorizer), Resources: features.NewResourcesClient(_env.Environment(), _env.SubscriptionID(), authorizer), - VirtualNetworks: network.NewVirtualNetworksClient(_env.Environment(), _env.SubscriptionID(), authorizer), Disks: compute.NewDisksClient(_env.Environment(), _env.SubscriptionID(), authorizer), DiskEncryptionSets: compute.NewDiskEncryptionSetsClient(_env.Environment(), _env.SubscriptionID(), authorizer), - Subnet: network.NewSubnetsClient(_env.Environment(), _env.SubscriptionID(), authorizer), - Interfaces: network.NewInterfacesClient(_env.Environment(), _env.SubscriptionID(), authorizer), - NetworkSecurityGroups: network.NewSecurityGroupsClient(_env.Environment(), _env.SubscriptionID(), authorizer), + Interfaces: interfacesClient, + LoadBalancers: loadBalancersClient, + NetworkSecurityGroups: securityGroupsClient, + Subnet: subnetsClient, + VirtualNetworks: virtualNetworksClient, Storage: storage.NewAccountsClient(_env.Environment(), _env.SubscriptionID(), authorizer), - LoadBalancers: network.NewLoadBalancersClient(_env.Environment(), _env.SubscriptionID(), authorizer), RestConfig: restconfig, HiveRestConfig: hiveRestConfig, diff --git a/test/e2e/update.go b/test/e2e/update.go index 7e34294e424..abb30fe8510 100644 --- a/test/e2e/update.go +++ b/test/e2e/update.go @@ -11,7 +11,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" "github.com/Azure/go-autorest/autorest/to" cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -115,10 +115,10 @@ var _ = Describe("Update cluster Managed Outbound IPs", func() { Expect(err).NotTo(HaveOccurred()) rgName = stringutils.LastTokenByte(*oc.ClusterProfile.ResourceGroupID, '/') - lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "") + resp, err := clients.LoadBalancers.Get(ctx, rgName, lbName, nil) Expect(err).NotTo(HaveOccurred()) - if getOutboundIPsCount(lb) != 1 { + if getOutboundIPsCount(resp.LoadBalancer) != 1 { By("sending the PATCH request to set ManagedOutboundIPs.Count to 1") err = clients.OpenshiftClusters.UpdateAndWait(ctx, vnetResourceGroup, clusterName, newManagedOutboundIPUpdateBody(1)) Expect(err).NotTo(HaveOccurred()) @@ -138,9 +138,9 @@ var _ = Describe("Update cluster Managed Outbound IPs", func() { Expect(*oc.NetworkProfile.LoadBalancerProfile.EffectiveOutboundIps).To(HaveLen(5)) By("checking outbound-rule-4 has required number IPs") - lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "") + resp, err := clients.LoadBalancers.Get(ctx, rgName, lbName, nil) Expect(err).NotTo(HaveOccurred()) - Expect(getOutboundIPsCount(lb)).To(Equal(5)) + Expect(getOutboundIPsCount(resp.LoadBalancer)).To(Equal(5)) By("sending the PUT request to decrease Managed Outbound IPs") oc.OpenShiftClusterProperties.NetworkProfile.LoadBalancerProfile.ManagedOutboundIps.Count = to.Int32Ptr(1) @@ -155,9 +155,9 @@ var _ = Describe("Update cluster Managed Outbound IPs", func() { Expect(*oc.NetworkProfile.LoadBalancerProfile.EffectiveOutboundIps).To(HaveLen(1)) By("checking outbound-rule-4 has required number of IPs") - lb, err = clients.LoadBalancers.Get(ctx, rgName, lbName, "") + resp, err = clients.LoadBalancers.Get(ctx, rgName, lbName, nil) Expect(err).NotTo(HaveOccurred()) - Expect(getOutboundIPsCount(lb)).To(Equal(1)) + Expect(getOutboundIPsCount(resp.LoadBalancer)).To(Equal(1)) }) }) @@ -183,11 +183,11 @@ func newManagedOutboundIPUpdateBody(managedOutboundIPCount int32) mgmtredhatopen } } -func getOutboundIPsCount(lb mgmtnetwork.LoadBalancer) int { +func getOutboundIPsCount(lb armnetwork.LoadBalancer) int { numOfIPs := 0 - for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { + for _, obRule := range lb.Properties.OutboundRules { if *obRule.Name == "outbound-rule-v4" { - numOfIPs = len(*obRule.FrontendIPConfigurations) + numOfIPs = len(obRule.Properties.FrontendIPConfigurations) } } return numOfIPs