SDK2: Remove deprecated network sdk from e2e. (#3859)

* subnets
* remove deprecated security group usage from e2e setup
* remove deprecated interface sdk from e2e.
* remove deprecated loadbalancer sdk from e2e.
* remove deprecated virtual networks sdk from e2e.
* remove deprecated subnets sdk from e2e

pkg/api/util/subnet/subnet.go

@@ -25,8 +25,7 @@ func Split(subnetID string) (string, string, error) {
 	return strings.Join(parts[:len(parts)-2], "/"), parts[len(parts)-1], nil
 }
 
-// NetworkSecurityGroupID returns the NetworkSecurityGroup ID for a given subnet
-// ID
+// NetworkSecurityGroupID returns the NetworkSecurityGroup ID for a given subnet ID
 func NetworkSecurityGroupID(oc *api.OpenShiftCluster, subnetID string) (string, error) {
 	infraID := oc.Properties.InfraID
 	if infraID == "" {

pkg/monitor/azure/nsg/nsg.go

@@ -75,10 +75,13 @@ func NewMonitor(log *logrus.Entry, oc *api.OpenShiftCluster, e env.Interface, su
 		return &monitoring.NoOpMonitor{Wg: wg}
 	}
 
-	options := arm.ClientOptions{
-		ClientOptions: e.Environment().ClientCertificateCredentialOptions().ClientOptions,
+	clientOptions := arm.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Cloud: e.Environment().Cloud,
+		},
 	}
-	client, err := armnetwork.NewSubnetsClient(subscriptionID, token, &options)
+
+	client, err := sdknetwork.NewSubnetsClient(subscriptionID, token, &clientOptions)
 	if err != nil {
 		log.Error("Unable to create the subnet client for NSG monitoring", err)
 		emitter.EmitGauge(MetricFailedNSGMonitorCreation, int64(1), dims)

pkg/util/azureclient/azuresdk/armnetwork/subnets.go

@@ -7,24 +7,23 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
-	sdknetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
 
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azcore"
 )
 
 // SubnetsClient is a minimal interface for azure-sdk-for-go subnets client
 type SubnetsClient interface {
-	Get(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *sdknetwork.SubnetsClientGetOptions) (sdknetwork.SubnetsClientGetResponse, error)
+	Get(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientGetOptions) (armnetwork.SubnetsClientGetResponse, error)
+	SubnetsClientAddons
 }
 
 type subnetsClient struct {
-	*sdknetwork.SubnetsClient
+	*armnetwork.SubnetsClient
 }
 
-var _ SubnetsClient = (*subnetsClient)(nil)
-
 func NewSubnetsClient(subscriptionID string, credential azcore.TokenCredential, options *arm.ClientOptions) (SubnetsClient, error) {
-	client, err := sdknetwork.NewSubnetsClient(subscriptionID, credential, options)
+	client, err := armnetwork.NewSubnetsClient(subscriptionID, credential, options)
 
-	return subnetsClient{client}, err
+	return &subnetsClient{client}, err
 }

pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go

@@ -0,0 +1,34 @@
+package armnetwork
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
+)
+
+// SubnetsClientAddons contains addons for SubnetsClient
+type SubnetsClientAddons interface {
+	CreateOrUpdateAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, subnetParameters armnetwork.Subnet, options *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) (err error)
+	DeleteAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientBeginDeleteOptions) error
+}
+
+func (c *subnetsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, subnetParameters armnetwork.Subnet, options *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) error {
+	poller, err := c.SubnetsClient.BeginCreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, subnetName, subnetParameters, options)
+	if err != nil {
+		return err
+	}
+	_, err = poller.PollUntilDone(ctx, nil)
+	return err
+}
+
+func (c *subnetsClient) DeleteAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientBeginDeleteOptions) error {
+	poller, err := c.SubnetsClient.BeginDelete(ctx, resourceGroupName, virtualNetworkName, subnetName, options)
+	if err != nil {
+		return err
+	}
+	_, err = poller.PollUntilDone(ctx, nil)
+	return err
+}

pkg/util/azureclient/azuresdk/armnetwork/virtualnetworks.go

@@ -0,0 +1,29 @@
+package armnetwork
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
+
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azcore"
+)
+
+// VirtualNetworksClient is a minimal interface for azure VirtualNetworksClient
+type VirtualNetworksClient interface {
+	Get(ctx context.Context, resourceGroupName string, virtualNetworkName string, options *armnetwork.VirtualNetworksClientGetOptions) (vnet armnetwork.VirtualNetworksClientGetResponse, err error)
+}
+
+type virtualNetworksClient struct {
+	*armnetwork.VirtualNetworksClient
+}
+
+// NewVirtualNetworksClient creates a new VirtualNetworksClient
+func NewVirtualNetworksClient(subscriptionID string, credential azcore.TokenCredential, options *arm.ClientOptions) (VirtualNetworksClient, error) {
+	client, err := armnetwork.NewVirtualNetworksClient(subscriptionID, credential, options)
+
+	return &virtualNetworksClient{client}, err
+}

test/e2e/adminapi_delete_managedresource.go

@@ -72,14 +72,16 @@ var _ = Describe("[Admin API] Delete managed resource action", func() {
 		lbName, err := getInfraID(ctx)
 		Expect(err).NotTo(HaveOccurred())
 
-		lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "")
+		lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, nil)
 		Expect(err).NotTo(HaveOccurred())
 
-		for _, fipConfig := range *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations {
-			if !strings.Contains(*fipConfig.PublicIPAddress.ID, "default-v4") && !strings.Contains(*fipConfig.PublicIPAddress.ID, "pip-v4") {
-				lbRuleID = *(*fipConfig.LoadBalancingRules)[0].ID
+		for _, fipConfig := range lb.Properties.FrontendIPConfigurations {
+			Expect(fipConfig.Properties.PublicIPAddress).NotTo(BeNil())
+			if !strings.Contains(*fipConfig.Properties.PublicIPAddress.ID, "default-v4") && !strings.Contains(*fipConfig.Properties.PublicIPAddress.ID, "pip-v4") {
+				Expect(fipConfig.Properties.LoadBalancingRules).To(HaveLen(1))
+				lbRuleID = *fipConfig.Properties.LoadBalancingRules[0].ID
 				fipConfigID = *fipConfig.ID
-				pipAddressID = *fipConfig.PublicIPAddress.ID
+				pipAddressID = *fipConfig.Properties.PublicIPAddress.ID
 			}
 		}
 

test/e2e/adminapi_resources.go

@@ -59,17 +59,16 @@ var _ = Describe("[Admin API] List Azure resources action", func() {
 			subnets[strings.ToLower(*p.SubnetID)] = struct{}{}
 		}
 
-		vnet, err := clients.VirtualNetworks.Get(ctx, r.ResourceGroup, r.ResourceName, "")
+		vnet, err := clients.VirtualNetworks.Get(ctx, r.ResourceGroup, r.ResourceName, nil)
 		Expect(err).NotTo(HaveOccurred())
 
-		for _, subnet := range *vnet.Subnets {
+		for _, subnet := range vnet.Properties.Subnets {
 			if _, ok := subnets[strings.ToLower(*subnet.ID)]; !ok {
 				continue
 			}
 
-			if subnet.SubnetPropertiesFormat != nil &&
-				subnet.RouteTable != nil {
-				expectedResourceIDs = append(expectedResourceIDs, strings.ToLower(*subnet.RouteTable.ID))
+			if subnet.Properties != nil && subnet.Properties.RouteTable != nil {
+				expectedResourceIDs = append(expectedResourceIDs, strings.ToLower(*subnet.Properties.RouteTable.ID))
 			}
 		}
 

test/e2e/cluster.go

@@ -11,7 +11,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
 	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-09-01/storage"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -94,35 +94,36 @@ var _ = Describe("Cluster", Serial, func() {
 				vnetR, err := azure.ParseResourceID(vnetID)
 				Expect(err).NotTo(HaveOccurred())
 
-				mgmtSubnet, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, "")
+				resp, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, nil)
 				Expect(err).NotTo(HaveOccurred())
+				subnet := resp.Subnet
 
-				if mgmtSubnet.SubnetPropertiesFormat == nil {
-					mgmtSubnet.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{}
+				if subnet.Properties == nil {
+					subnet.Properties = &armnetwork.SubnetPropertiesFormat{}
 				}
 
-				if mgmtSubnet.SubnetPropertiesFormat.ServiceEndpoints == nil {
-					mgmtSubnet.SubnetPropertiesFormat.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{}
+				if subnet.Properties.ServiceEndpoints == nil {
+					subnet.Properties.ServiceEndpoints = []*armnetwork.ServiceEndpointPropertiesFormat{}
 				}
 
 				// Check whether service endpoint is already there before trying to add
 				// it; trying to add a duplicate results in an error
-				for _, se := range *mgmtSubnet.ServiceEndpoints {
+				for _, se := range subnet.Properties.ServiceEndpoints {
 					if se.Service != nil && *se.Service == "Microsoft.Storage" {
 						subnetAlreadyHasStorageEndpoint = true
 						break
 					}
 				}
 
 				if !subnetAlreadyHasStorageEndpoint {
-					storageEndpoint := mgmtnetwork.ServiceEndpointPropertiesFormat{
+					storageEndpoint := armnetwork.ServiceEndpointPropertiesFormat{
 						Service:   to.StringPtr("Microsoft.Storage"),
-						Locations: &[]string{"*"},
+						Locations: []*string{to.StringPtr("*")},
 					}
 
-					*mgmtSubnet.ServiceEndpoints = append(*mgmtSubnet.ServiceEndpoints, storageEndpoint)
+					subnet.Properties.ServiceEndpoints = append(subnet.Properties.ServiceEndpoints, &storageEndpoint)
 
-					err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, mgmtSubnet)
+					err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, subnet, nil)
 					Expect(err).NotTo(HaveOccurred())
 				}
 			}
@@ -193,16 +194,17 @@ var _ = Describe("Cluster", Serial, func() {
 					vnetR, err := azure.ParseResourceID(vnetID)
 					Expect(err).NotTo(HaveOccurred())
 
-					mgmtSubnet, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, "")
+					resp, err := clients.Subnet.Get(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, nil)
 					Expect(err).NotTo(HaveOccurred())
+					subnet := resp.Subnet
 
-					if mgmtSubnet.SubnetPropertiesFormat == nil {
-						mgmtSubnet.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{}
+					if subnet.Properties == nil {
+						subnet.Properties = &armnetwork.SubnetPropertiesFormat{}
 					}
 
-					mgmtSubnet.SubnetPropertiesFormat.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{}
+					subnet.Properties.ServiceEndpoints = []*armnetwork.ServiceEndpointPropertiesFormat{}
 
-					err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, mgmtSubnet)
+					err = clients.Subnet.CreateOrUpdateAndWait(ctx, vnetResourceGroup, vnetR.ResourceName, subnetName, subnet, nil)
 					Expect(err).NotTo(HaveOccurred())
 				}
 			}

test/e2e/dns.go

@@ -20,7 +20,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork"
 	"github.com/Azure/ARO-RP/pkg/util/ready"
 	"github.com/Azure/ARO-RP/pkg/util/stringutils"
 )
@@ -77,15 +77,15 @@ var _ = Describe("ARO cluster DNS", func() {
 
 		clusterResourceGroup := stringutils.LastTokenByte(*oc.OpenShiftClusterProperties.ClusterProfile.ResourceGroupID, '/')
 		for wn := range workerNodes {
-			nic, err := clients.Interfaces.Get(ctx, clusterResourceGroup, nicName(wn), "")
+			resp, err := clients.Interfaces.Get(ctx, clusterResourceGroup, nicName(wn), nil)
 			Expect(err).NotTo(HaveOccurred())
+			nic := resp.Interface
 
-			Expect(nic.InterfacePropertiesFormat).NotTo(BeNil())
-			Expect(nic.IPConfigurations).NotTo(BeNil())
-			Expect(*nic.IPConfigurations).To(HaveLen(1))
-			Expect((*nic.IPConfigurations)[0].InterfaceIPConfigurationPropertiesFormat).NotTo(BeNil())
-			Expect((*nic.IPConfigurations)[0].PrivateIPAddress).NotTo(BeNil())
-			workerNodes[wn] = *(*nic.IPConfigurations)[0].PrivateIPAddress
+			Expect(nic.Properties).NotTo(BeNil())
+			Expect(nic.Properties.IPConfigurations).To(HaveLen(1))
+			Expect(nic.Properties.IPConfigurations[0].Properties).NotTo(BeNil())
+			Expect(nic.Properties.IPConfigurations[0].Properties.PrivateIPAddress).NotTo(BeNil())
+			workerNodes[wn] = *nic.Properties.IPConfigurations[0].Properties.PrivateIPAddress
 		}
 
 		By("preparing to read resolv.conf from each of the worker nodes by allowing the test namespace's ServiceAccount to use the hostmount-anyuid SecurityContextConstraint")
@@ -380,14 +380,18 @@ func verifyResolvConf(
 	return nil
 }
 
-func toggleAcceleratedNetworking(ctx context.Context, interfaces network.InterfacesClient, clusterResourceGroup string, nodeName string, enabled bool) error {
-	nic, err := interfaces.Get(ctx, clusterResourceGroup, nicName(nodeName), "")
+func toggleAcceleratedNetworking(ctx context.Context, interfaces armnetwork.InterfacesClient, clusterResourceGroup string, nodeName string, enabled bool) error {
+	resp, err := interfaces.Get(ctx, clusterResourceGroup, nicName(nodeName), nil)
 	if err != nil {
 		return err
 	}
+	nic := resp.Interface
 
-	nic.EnableAcceleratedNetworking = to.BoolPtr(enabled)
-	err = clients.Interfaces.CreateOrUpdateAndWait(ctx, clusterResourceGroup, nicName(nodeName), nic)
+	if nic.Properties == nil {
+		return fmt.Errorf("NIC properties are nil")
+	}
+	nic.Properties.EnableAcceleratedNetworking = to.BoolPtr(enabled)
+	err = clients.Interfaces.CreateOrUpdateAndWait(ctx, clusterResourceGroup, nicName(nodeName), nic, nil)
 	return err
 }