From 69378fb634f2bd861d6d35e1aad8ed58f63dc410 Mon Sep 17 00:00:00 2001 From: Ayato Tokubi Date: Fri, 4 Oct 2024 18:35:30 +0100 Subject: [PATCH] SDK2: Replace SDK to new one in util/cluster (#3884) * add subnets.List * remove deprecated nsg & subnets client from util/cluster * add routetable client * remove deprecated route tables client from util/cluster * add virtual network peerings client * remove deprecated virtual network peerings from util/cluster * remove the rest usage of mgmt network sdk * remove unused sdk wrappers --- .../azuresdk/armnetwork/routetable.go | 32 ++++ .../azuresdk/armnetwork/routetable_addons.go | 24 +++ .../azuresdk/armnetwork/subnets_addons.go | 14 ++ .../armnetwork/virtualnetworkpeerings.go | 28 +++ .../virtualnetworkpeerings_addons.go | 33 ++++ pkg/util/azureclient/mgmt/network/generate.go | 2 +- .../mgmt/network/securitygroups.go | 36 ---- .../mgmt/network/securitygroups_addons.go | 50 ------ .../mgmt/network/virtualnetworkpeerings.go | 36 ---- .../network/virtualnetworkpeerings_addons.go | 33 ---- pkg/util/cluster/cluster.go | 127 +++++++------- .../azuresdk/armnetwork/armnetwork.go | 15 ++ .../mocks/azureclient/mgmt/network/network.go | 166 +----------------- 13 files changed, 217 insertions(+), 379 deletions(-) create mode 100644 pkg/util/azureclient/azuresdk/armnetwork/routetable.go create mode 100644 pkg/util/azureclient/azuresdk/armnetwork/routetable_addons.go create mode 100644 pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings.go create mode 100644 pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings_addons.go delete mode 100644 pkg/util/azureclient/mgmt/network/securitygroups.go delete mode 100644 pkg/util/azureclient/mgmt/network/securitygroups_addons.go delete mode 100644 pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go delete mode 100644 pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go diff --git a/pkg/util/azureclient/azuresdk/armnetwork/routetable.go b/pkg/util/azureclient/azuresdk/armnetwork/routetable.go new file mode 100644 index 00000000000..8bd7319ebbe --- /dev/null +++ b/pkg/util/azureclient/azuresdk/armnetwork/routetable.go @@ -0,0 +1,32 @@ +package armnetwork + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" + + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azcore" +) + +// RouteTablesClient is a minimal interface for azure RouteTablesClient +type RouteTablesClient interface { + Get(ctx context.Context, resourceGroupName string, routeTableName string, options *armnetwork.RouteTablesClientGetOptions) (result armnetwork.RouteTablesClientGetResponse, err error) + RouteTablesClientAddons +} + +type routeTablesClient struct { + *armnetwork.RouteTablesClient +} + +// NewRouteTablesClient creates a new RouteTablesClient +func NewRouteTablesClient(subscriptionID string, credential azcore.TokenCredential, options *arm.ClientOptions) (RouteTablesClient, error) { + clientFactory, err := armnetwork.NewClientFactory(subscriptionID, credential, options) + if err != nil { + return nil, err + } + return &routeTablesClient{clientFactory.NewRouteTablesClient()}, err +} diff --git a/pkg/util/azureclient/azuresdk/armnetwork/routetable_addons.go b/pkg/util/azureclient/azuresdk/armnetwork/routetable_addons.go new file mode 100644 index 00000000000..717c69c481d --- /dev/null +++ b/pkg/util/azureclient/azuresdk/armnetwork/routetable_addons.go @@ -0,0 +1,24 @@ +package armnetwork + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" +) + +// RouteTablesClientAddons contains addons for RouteTablesClient +type RouteTablesClientAddons interface { + DeleteAndWait(ctx context.Context, resourceGroupName string, routeTableName string, options *armnetwork.RouteTablesClientBeginDeleteOptions) error +} + +func (c *routeTablesClient) DeleteAndWait(ctx context.Context, resourceGroupName string, routeTableName string, options *armnetwork.RouteTablesClientBeginDeleteOptions) error { + poller, err := c.RouteTablesClient.BeginDelete(ctx, resourceGroupName, routeTableName, options) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + return err +} diff --git a/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go b/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go index 57fe37407c5..4203c9183e9 100644 --- a/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go +++ b/pkg/util/azureclient/azuresdk/armnetwork/subnets_addons.go @@ -13,6 +13,7 @@ import ( type SubnetsClientAddons interface { CreateOrUpdateAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, subnetParameters armnetwork.Subnet, options *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) (err error) DeleteAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, options *armnetwork.SubnetsClientBeginDeleteOptions) error + List(ctx context.Context, resourceGroupName, virtualNetworkName string, options *armnetwork.SubnetsClientListOptions) ([]*armnetwork.Subnet, error) } func (c *subnetsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName, virtualNetworkName, subnetName string, subnetParameters armnetwork.Subnet, options *armnetwork.SubnetsClientBeginCreateOrUpdateOptions) error { @@ -32,3 +33,16 @@ func (c *subnetsClient) DeleteAndWait(ctx context.Context, resourceGroupName, vi _, err = poller.PollUntilDone(ctx, nil) return err } + +func (c *subnetsClient) List(ctx context.Context, resourceGroupName, virtualNetworkName string, options *armnetwork.SubnetsClientListOptions) (result []*armnetwork.Subnet, err error) { + pager := c.SubnetsClient.NewListPager(resourceGroupName, virtualNetworkName, options) + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + result = append(result, page.Value...) + } + return result, nil +} diff --git a/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings.go b/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings.go new file mode 100644 index 00000000000..b3d9d4e7883 --- /dev/null +++ b/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings.go @@ -0,0 +1,28 @@ +package armnetwork + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" + + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/azcore" +) + +type VirtualNetworkPeeringsClient interface { + VirtualNetworkPeeringsAddons +} + +type virtualNetworkPeeringsClient struct { + *armnetwork.VirtualNetworkPeeringsClient +} + +// NewVirtualNetworkPeeringsClient creates a new VirtualNetworkPeeringsClient +func NewVirtualNetworkPeeringsClient(subscriptionID string, credential azcore.TokenCredential, options *arm.ClientOptions) (VirtualNetworkPeeringsClient, error) { + clientFactory, err := armnetwork.NewClientFactory(subscriptionID, credential, options) + if err != nil { + return nil, err + } + return &virtualNetworkPeeringsClient{clientFactory.NewVirtualNetworkPeeringsClient()}, err +} diff --git a/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings_addons.go b/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings_addons.go new file mode 100644 index 00000000000..0cfb8d53d44 --- /dev/null +++ b/pkg/util/azureclient/azuresdk/armnetwork/virtualnetworkpeerings_addons.go @@ -0,0 +1,33 @@ +package armnetwork + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" +) + +type VirtualNetworkPeeringsAddons interface { + CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters armnetwork.VirtualNetworkPeering, options *armnetwork.VirtualNetworkPeeringsClientBeginCreateOrUpdateOptions) (err error) + DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, options *armnetwork.VirtualNetworkPeeringsClientBeginDeleteOptions) (err error) +} + +func (c *virtualNetworkPeeringsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters armnetwork.VirtualNetworkPeering, options *armnetwork.VirtualNetworkPeeringsClientBeginCreateOrUpdateOptions) (err error) { + poller, err := c.VirtualNetworkPeeringsClient.BeginCreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName, virtualNetworkPeeringParameters, options) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + return err +} + +func (c *virtualNetworkPeeringsClient) DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, options *armnetwork.VirtualNetworkPeeringsClientBeginDeleteOptions) (err error) { + poller, err := c.VirtualNetworkPeeringsClient.BeginDelete(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName, options) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + return err +} diff --git a/pkg/util/azureclient/mgmt/network/generate.go b/pkg/util/azureclient/mgmt/network/generate.go index c43ef8886c4..8f32bf26402 100644 --- a/pkg/util/azureclient/mgmt/network/generate.go +++ b/pkg/util/azureclient/mgmt/network/generate.go @@ -4,5 +4,5 @@ package network // Licensed under the Apache License 2.0. //go:generate rm -rf ../../../../util/mocks/$GOPACKAGE -//go:generate mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE InterfacesClient,LoadBalancersClient,PrivateEndpointsClient,PublicIPAddressesClient,LoadBalancerBackendAddressPoolsClient,RouteTablesClient,SubnetsClient,VirtualNetworksClient,SecurityGroupsClient,VirtualNetworkPeeringsClient,UsageClient,FlowLogsClient +//go:generate mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE InterfacesClient,LoadBalancersClient,PrivateEndpointsClient,PublicIPAddressesClient,LoadBalancerBackendAddressPoolsClient,RouteTablesClient,SubnetsClient,VirtualNetworksClient,UsageClient,FlowLogsClient //go:generate goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/util/azureclient/mgmt/network/securitygroups.go b/pkg/util/azureclient/mgmt/network/securitygroups.go deleted file mode 100644 index fa3546dd752..00000000000 --- a/pkg/util/azureclient/mgmt/network/securitygroups.go +++ /dev/null @@ -1,36 +0,0 @@ -package network - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" - "github.com/Azure/go-autorest/autorest" - - "github.com/Azure/ARO-RP/pkg/util/azureclient" -) - -// SecurityGroupsClient is a minimal interface for azure SecurityGroupsClient -type SecurityGroupsClient interface { - Get(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, expand string) (result mgmtnetwork.SecurityGroup, err error) - SecurityGroupsClientAddons -} - -type securityGroupsClient struct { - mgmtnetwork.SecurityGroupsClient -} - -var _ SecurityGroupsClient = &securityGroupsClient{} - -// NewSecurityGroupsClient creates a new SecurityGroupsClient -func NewSecurityGroupsClient(environment *azureclient.AROEnvironment, subscriptionID string, authorizer autorest.Authorizer) SecurityGroupsClient { - client := mgmtnetwork.NewSecurityGroupsClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID) - client.Authorizer = authorizer - client.Sender = azureclient.DecorateSenderWithLogging(client.Sender) - - return &securityGroupsClient{ - SecurityGroupsClient: client, - } -} diff --git a/pkg/util/azureclient/mgmt/network/securitygroups_addons.go b/pkg/util/azureclient/mgmt/network/securitygroups_addons.go deleted file mode 100644 index 1ef9fc64557..00000000000 --- a/pkg/util/azureclient/mgmt/network/securitygroups_addons.go +++ /dev/null @@ -1,50 +0,0 @@ -package network - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" -) - -// SecurityGroupsClientAddons contains addons for SecurityGroupsClient -type SecurityGroupsClientAddons interface { - CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, parameters mgmtnetwork.SecurityGroup) (err error) - DeleteAndWait(ctx context.Context, resourceGroupName string, networkSecurityGroupName string) (err error) - List(ctx context.Context, resourceGroupName string) (result []mgmtnetwork.SecurityGroup, err error) -} - -func (c *securityGroupsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, networkSecurityGroupName string, parameters mgmtnetwork.SecurityGroup) (err error) { - future, err := c.SecurityGroupsClient.CreateOrUpdate(ctx, resourceGroupName, networkSecurityGroupName, parameters) - if err != nil { - return err - } - return future.WaitForCompletionRef(ctx, c.Client) -} - -func (c *securityGroupsClient) DeleteAndWait(ctx context.Context, resourceGroupName string, networkSecurityGroupName string) (err error) { - future, err := c.SecurityGroupsClient.Delete(ctx, resourceGroupName, networkSecurityGroupName) - if err != nil { - return err - } - return future.WaitForCompletionRef(ctx, c.Client) -} - -func (c *securityGroupsClient) List(ctx context.Context, resourceGroupName string) (result []mgmtnetwork.SecurityGroup, err error) { - page, err := c.SecurityGroupsClient.List(ctx, resourceGroupName) - if err != nil { - return nil, err - } - - for page.NotDone() { - result = append(result, page.Values()...) - - err = page.NextWithContext(ctx) - if err != nil { - return nil, err - } - } - return result, nil -} diff --git a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go b/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go deleted file mode 100644 index 81b664c84cc..00000000000 --- a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings.go +++ /dev/null @@ -1,36 +0,0 @@ -package network - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" - "github.com/Azure/go-autorest/autorest" - - "github.com/Azure/ARO-RP/pkg/util/azureclient" -) - -type VirtualNetworkPeeringsClient interface { - CreateOrUpdate(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (result mgmtnetwork.VirtualNetworkPeeringsCreateOrUpdateFuture, err error) - Delete(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (result mgmtnetwork.VirtualNetworkPeeringsDeleteFuture, err error) - VirtualNetworkPeeringsAddons -} - -type virtualNetworkPeeringsClient struct { - mgmtnetwork.VirtualNetworkPeeringsClient -} - -var _ VirtualNetworkPeeringsClient = &virtualNetworkPeeringsClient{} - -// NewVirtualNetworkPeeringsClient creates a new VirtualNetworkPeeringsClient -func NewVirtualNetworkPeeringsClient(environment *azureclient.AROEnvironment, subscriptionID string, authorizer autorest.Authorizer) VirtualNetworkPeeringsClient { - client := mgmtnetwork.NewVirtualNetworkPeeringsClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID) - client.Authorizer = authorizer - client.Sender = azureclient.DecorateSenderWithLogging(client.Sender) - - return &virtualNetworkPeeringsClient{ - VirtualNetworkPeeringsClient: client, - } -} diff --git a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go b/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go deleted file mode 100644 index 3ffd88887df..00000000000 --- a/pkg/util/azureclient/mgmt/network/virtualnetworkpeerings_addons.go +++ /dev/null @@ -1,33 +0,0 @@ -package network - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" -) - -type VirtualNetworkPeeringsAddons interface { - CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (err error) - DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (err error) -} - -func (c *virtualNetworkPeeringsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string, virtualNetworkPeeringParameters mgmtnetwork.VirtualNetworkPeering) (err error) { - future, err := c.CreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName, virtualNetworkPeeringParameters) - if err != nil { - return err - } - - return future.WaitForCompletionRef(ctx, c.Client) -} - -func (c *virtualNetworkPeeringsClient) DeleteAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, virtualNetworkPeeringName string) (err error) { - future, err := c.VirtualNetworkPeeringsClient.Delete(ctx, resourceGroupName, virtualNetworkName, virtualNetworkPeeringName) - if err != nil { - return err - } - - return future.WaitForCompletionRef(ctx, c.Client) -} diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go index 589f8e80b90..21c61535e08 100644 --- a/pkg/util/cluster/cluster.go +++ b/pkg/util/cluster/cluster.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" sdkkeyvault "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault" - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + sdknetwork "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2" mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization" mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features" "github.com/Azure/go-autorest/autorest" @@ -30,6 +30,7 @@ import ( "github.com/jongio/azidext/go/azidext" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/deploy/assets" @@ -37,9 +38,9 @@ import ( "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armkeyvault" + "github.com/Azure/ARO-RP/pkg/util/azureclient/azuresdk/armnetwork" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" - "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/azureerrors" utilgraph "github.com/Azure/ARO-RP/pkg/util/graph" "github.com/Azure/ARO-RP/pkg/util/rbac" @@ -58,12 +59,12 @@ type Cluster struct { deployments features.DeploymentsClient groups features.ResourceGroupsClient openshiftclusters InternalClient - securitygroups network.SecurityGroupsClient - subnets network.SubnetsClient - routetables network.RouteTablesClient + securitygroups armnetwork.SecurityGroupsClient + subnets armnetwork.SubnetsClient + routetables armnetwork.RouteTablesClient roleassignments authorization.RoleAssignmentsClient - peerings network.VirtualNetworkPeeringsClient - ciParentVnetPeerings network.VirtualNetworkPeeringsClient + peerings armnetwork.VirtualNetworkPeeringsClient + ciParentVnetPeerings armnetwork.VirtualNetworkPeeringsClient vaultsClient armkeyvault.VaultsClient } @@ -97,11 +98,33 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) { }, } + clientOptions := environment.Environment().ArmClientOptions() + vaultClient, err := armkeyvault.NewVaultsClient(environment.SubscriptionID(), spTokenCredential, &armOption) + if err != nil { + return nil, err + } + + securityGroupsClient, err := armnetwork.NewSecurityGroupsClient(environment.SubscriptionID(), spTokenCredential, clientOptions) + if err != nil { + return nil, err + } + subnetsClient, err := armnetwork.NewSubnetsClient(environment.SubscriptionID(), spTokenCredential, clientOptions) if err != nil { return nil, err } + + routeTablesClient, err := armnetwork.NewRouteTablesClient(environment.SubscriptionID(), spTokenCredential, clientOptions) + if err != nil { + return nil, err + } + + virtualNetworkPeeringsClient, err := armnetwork.NewVirtualNetworkPeeringsClient(environment.SubscriptionID(), spTokenCredential, clientOptions) + if err != nil { + return nil, err + } + c := &Cluster{ log: log, env: environment, @@ -111,11 +134,11 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) { deployments: features.NewDeploymentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), groups: features.NewResourceGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), openshiftclusters: NewInternalClient(log, environment, authorizer), - securitygroups: network.NewSecurityGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - subnets: network.NewSubnetsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - routetables: network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer), + securitygroups: securityGroupsClient, + subnets: subnetsClient, + routetables: routeTablesClient, roleassignments: authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), - peerings: network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + peerings: virtualNetworkPeeringsClient, vaultsClient: vaultClient, } @@ -128,7 +151,12 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) { return nil, err } - c.ciParentVnetPeerings = network.NewVirtualNetworkPeeringsClient(environment.Environment(), r.SubscriptionID, authorizer) + ciVirtualNetworkPeeringsClient, err := armnetwork.NewVirtualNetworkPeeringsClient(r.SubscriptionID, spTokenCredential, clientOptions) + if err != nil { + return nil, err + } + + c.ciParentVnetPeerings = ciVirtualNetworkPeeringsClient } return c, nil @@ -246,13 +274,13 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str // TODO: ick if os.Getenv("NO_INTERNET") != "" { parameters["routes"] = &arm.ParametersParameter{ - Value: []mgmtnetwork.Route{ + Value: []sdknetwork.Route{ { - RoutePropertiesFormat: &mgmtnetwork.RoutePropertiesFormat{ - AddressPrefix: to.StringPtr("0.0.0.0/0"), - NextHopType: mgmtnetwork.RouteNextHopTypeNone, + Properties: &sdknetwork.RoutePropertiesFormat{ + AddressPrefix: ptr.To("0.0.0.0/0"), + NextHopType: ptr.To(sdknetwork.RouteNextHopTypeNone), }, - Name: to.StringPtr("blackhole"), + Name: ptr.To("blackhole"), }, }, } @@ -371,21 +399,21 @@ func ipRangesContainCIDR(ipRanges []*net.IPNet, cidr string) (bool, error) { // Because an az subnet can cover multiple ipranges, we need to return a slice // instead of just a single ip range. This function never errors. If something // goes wrong, it instead returns an empty list. -func GetIPRangesFromSubnet(subnet mgmtnetwork.Subnet) []*net.IPNet { +func GetIPRangesFromSubnet(subnet *sdknetwork.Subnet) []*net.IPNet { ipRanges := []*net.IPNet{} - if subnet.AddressPrefix != nil { - _, ipRange, err := net.ParseCIDR(*subnet.AddressPrefix) + if subnet.Properties.AddressPrefix != nil { + _, ipRange, err := net.ParseCIDR(*subnet.Properties.AddressPrefix) if err == nil { ipRanges = append(ipRanges, ipRange) } } - if subnet.AddressPrefixes == nil { + if subnet.Properties.AddressPrefixes == nil { return ipRanges } - for _, snetPrefix := range *subnet.AddressPrefixes { - _, ipRange, err := net.ParseCIDR(snetPrefix) + for _, snetPrefix := range subnet.Properties.AddressPrefixes { + _, ipRange, err := net.ParseCIDR(*snetPrefix) if err == nil { ipRanges = append(ipRanges, ipRange) } @@ -393,26 +421,6 @@ func GetIPRangesFromSubnet(subnet mgmtnetwork.Subnet) []*net.IPNet { return ipRanges } -// getAllDevSubnets queries azure to retrieve all subnets assigned the vnet -// `dev-vnet` in the current resource group -func (c *Cluster) getAllDevSubnets() ([]mgmtnetwork.Subnet, error) { - allSubnets := []mgmtnetwork.Subnet{} - availSnetResults, err := c.subnets.List(context.Background(), c.env.ResourceGroup(), "dev-vnet") - if err != nil { - return allSubnets, err - } - allSubnets = append(allSubnets, availSnetResults.Values()...) - for availSnetResults.NotDone() { - err = availSnetResults.NextWithContext(context.Background()) - if err != nil { - break - } - allSubnets = append(allSubnets, availSnetResults.Values()...) - } - - return allSubnets, nil -} - func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, workerSubnet string, err error) { // pick a random 23 in range [10.3.0.0, 10.127.255.0], making sure it doesn't // conflict with other subnets present in out dev-vnet @@ -420,7 +428,7 @@ func (c *Cluster) generateSubnets() (vnetPrefix string, masterSubnet string, wor // 10.1.0.0/24 is used by rp-vnet to host Proxy VM // 10.2.0.0/24 is used by dev-vpn-vnet to host VirtualNetworkGateway - allSubnets, err := c.getAllDevSubnets() + allSubnets, err := c.subnets.List(context.Background(), c.env.ResourceGroup(), "dev-vnet", nil) if err != nil { c.log.Warnf("Error getting existing subnets. Continuing regardless: %v", err) } @@ -724,10 +732,10 @@ func (c *Cluster) fixupNSGs(ctx context.Context, vnetResourceGroup, clusterName // very occasionally c.securitygroups.List returns an empty list in // production. No idea why. Let's try retrying it... - var nsgs []mgmtnetwork.SecurityGroup + var nsgs []*sdknetwork.SecurityGroup err := wait.PollImmediateUntil(10*time.Second, func() (bool, error) { var err error - nsgs, err = c.securitygroups.List(ctx, "aro-"+clusterName) + nsgs, err = c.securitygroups.List(ctx, "aro-"+clusterName, nil) return len(nsgs) > 0, err }, timeoutCtx.Done()) if err != nil { @@ -735,16 +743,17 @@ func (c *Cluster) fixupNSGs(ctx context.Context, vnetResourceGroup, clusterName } for _, subnetName := range []string{clusterName + "-master", clusterName + "-worker"} { - subnet, err := c.subnets.Get(ctx, vnetResourceGroup, "dev-vnet", subnetName, "") + resp, err := c.subnets.Get(ctx, vnetResourceGroup, "dev-vnet", subnetName, nil) if err != nil { return err } + subnet := resp.Subnet - subnet.NetworkSecurityGroup = &mgmtnetwork.SecurityGroup{ + subnet.Properties.NetworkSecurityGroup = &sdknetwork.SecurityGroup{ ID: nsgs[0].ID, } - err = c.subnets.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", subnetName, subnet) + err = c.subnets.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", subnetName, subnet, nil) if err != nil { return err } @@ -829,7 +838,7 @@ func (c *Cluster) deleteResourceGroup(ctx context.Context, resourceGroup string) func (c *Cluster) deleteVnetPeerings(ctx context.Context, resourceGroup string) error { r, err := azure.ParseResourceID(c.ciParentVnet) if err == nil { - err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, resourceGroup+"-peer") + err = c.ciParentVnetPeerings.DeleteAndWait(ctx, r.ResourceGroup, r.ResourceName, resourceGroup+"-peer", nil) } if err != nil { return fmt.Errorf("error deleting vnet peerings: %w", err) @@ -850,18 +859,18 @@ func (c *Cluster) deleteVnetResources(ctx context.Context, resourceGroup, vnetNa var errs []error c.log.Info("deleting master/worker subnets") - if err := c.subnets.DeleteAndWait(ctx, resourceGroup, vnetName, clusterName+"-master"); err != nil { + if err := c.subnets.DeleteAndWait(ctx, resourceGroup, vnetName, clusterName+"-master", nil); err != nil { c.log.Errorf("error when deleting master subnet: %v", err) errs = append(errs, err) } - if err := c.subnets.DeleteAndWait(ctx, resourceGroup, vnetName, clusterName+"-worker"); err != nil { + if err := c.subnets.DeleteAndWait(ctx, resourceGroup, vnetName, clusterName+"-worker", nil); err != nil { c.log.Errorf("error when deleting worker subnet: %v", err) errs = append(errs, err) } c.log.Info("deleting route table") - if err := c.routetables.DeleteAndWait(ctx, resourceGroup, clusterName+"-rt"); err != nil { + if err := c.routetables.DeleteAndWait(ctx, resourceGroup, clusterName+"-rt", nil); err != nil { c.log.Errorf("error when deleting route table: %v", err) errs = append(errs, err) } @@ -877,16 +886,16 @@ func (c *Cluster) peerSubnetsToCI(ctx context.Context, vnetResourceGroup, cluste return err } - clusterProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{ - RemoteVirtualNetwork: &mgmtnetwork.SubResource{ + clusterProp := &sdknetwork.VirtualNetworkPeeringPropertiesFormat{ + RemoteVirtualNetwork: &sdknetwork.SubResource{ ID: &c.ciParentVnet, }, AllowVirtualNetworkAccess: to.BoolPtr(true), AllowForwardedTraffic: to.BoolPtr(true), UseRemoteGateways: to.BoolPtr(true), } - rpProp := &mgmtnetwork.VirtualNetworkPeeringPropertiesFormat{ - RemoteVirtualNetwork: &mgmtnetwork.SubResource{ + rpProp := &sdknetwork.VirtualNetworkPeeringPropertiesFormat{ + RemoteVirtualNetwork: &sdknetwork.SubResource{ ID: &cluster, }, AllowVirtualNetworkAccess: to.BoolPtr(true), @@ -894,12 +903,12 @@ func (c *Cluster) peerSubnetsToCI(ctx context.Context, vnetResourceGroup, cluste AllowGatewayTransit: to.BoolPtr(true), } - err = c.peerings.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", r.ResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: clusterProp}) + err = c.peerings.CreateOrUpdateAndWait(ctx, vnetResourceGroup, "dev-vnet", r.ResourceGroup+"-peer", sdknetwork.VirtualNetworkPeering{Properties: clusterProp}, nil) if err != nil { return err } - err = c.ciParentVnetPeerings.CreateOrUpdateAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer", mgmtnetwork.VirtualNetworkPeering{VirtualNetworkPeeringPropertiesFormat: rpProp}) + err = c.ciParentVnetPeerings.CreateOrUpdateAndWait(ctx, r.ResourceGroup, r.ResourceName, vnetResourceGroup+"-peer", sdknetwork.VirtualNetworkPeering{Properties: rpProp}, nil) if err != nil { return err } diff --git a/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go b/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go index d3e41306636..a9b96b77b08 100644 --- a/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go +++ b/pkg/util/mocks/azureclient/azuresdk/armnetwork/armnetwork.go @@ -548,3 +548,18 @@ func (mr *MockSubnetsClientMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 any) * mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockSubnetsClient)(nil).Get), arg0, arg1, arg2, arg3, arg4) } + +// List mocks base method. +func (m *MockSubnetsClient) List(arg0 context.Context, arg1, arg2 string, arg3 *armnetwork.SubnetsClientListOptions) ([]*armnetwork.Subnet, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].([]*armnetwork.Subnet) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockSubnetsClientMockRecorder) List(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockSubnetsClient)(nil).List), arg0, arg1, arg2, arg3) +} diff --git a/pkg/util/mocks/azureclient/mgmt/network/network.go b/pkg/util/mocks/azureclient/mgmt/network/network.go index 7d9e3dbe5e8..5515a6ca80a 100644 --- a/pkg/util/mocks/azureclient/mgmt/network/network.go +++ b/pkg/util/mocks/azureclient/mgmt/network/network.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network (interfaces: InterfacesClient,LoadBalancersClient,PrivateEndpointsClient,PublicIPAddressesClient,LoadBalancerBackendAddressPoolsClient,RouteTablesClient,SubnetsClient,VirtualNetworksClient,SecurityGroupsClient,VirtualNetworkPeeringsClient,UsageClient,FlowLogsClient) +// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network (interfaces: InterfacesClient,LoadBalancersClient,PrivateEndpointsClient,PublicIPAddressesClient,LoadBalancerBackendAddressPoolsClient,RouteTablesClient,SubnetsClient,VirtualNetworksClient,UsageClient,FlowLogsClient) // // Generated by this command: // -// mockgen -destination=../../../../util/mocks/azureclient/mgmt/network/network.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network InterfacesClient,LoadBalancersClient,PrivateEndpointsClient,PublicIPAddressesClient,LoadBalancerBackendAddressPoolsClient,RouteTablesClient,SubnetsClient,VirtualNetworksClient,SecurityGroupsClient,VirtualNetworkPeeringsClient,UsageClient,FlowLogsClient +// mockgen -destination=../../../../util/mocks/azureclient/mgmt/network/network.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network InterfacesClient,LoadBalancersClient,PrivateEndpointsClient,PublicIPAddressesClient,LoadBalancerBackendAddressPoolsClient,RouteTablesClient,SubnetsClient,VirtualNetworksClient,UsageClient,FlowLogsClient // // Package mock_network is a generated GoMock package. @@ -491,168 +491,6 @@ func (mr *MockVirtualNetworksClientMockRecorder) Get(arg0, arg1, arg2, arg3 any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockVirtualNetworksClient)(nil).Get), arg0, arg1, arg2, arg3) } -// MockSecurityGroupsClient is a mock of SecurityGroupsClient interface. -type MockSecurityGroupsClient struct { - ctrl *gomock.Controller - recorder *MockSecurityGroupsClientMockRecorder -} - -// MockSecurityGroupsClientMockRecorder is the mock recorder for MockSecurityGroupsClient. -type MockSecurityGroupsClientMockRecorder struct { - mock *MockSecurityGroupsClient -} - -// NewMockSecurityGroupsClient creates a new mock instance. -func NewMockSecurityGroupsClient(ctrl *gomock.Controller) *MockSecurityGroupsClient { - mock := &MockSecurityGroupsClient{ctrl: ctrl} - mock.recorder = &MockSecurityGroupsClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockSecurityGroupsClient) EXPECT() *MockSecurityGroupsClientMockRecorder { - return m.recorder -} - -// CreateOrUpdateAndWait mocks base method. -func (m *MockSecurityGroupsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2 string, arg3 network.SecurityGroup) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait. -func (mr *MockSecurityGroupsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockSecurityGroupsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3) -} - -// DeleteAndWait mocks base method. -func (m *MockSecurityGroupsClient) DeleteAndWait(arg0 context.Context, arg1, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAndWait indicates an expected call of DeleteAndWait. -func (mr *MockSecurityGroupsClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockSecurityGroupsClient)(nil).DeleteAndWait), arg0, arg1, arg2) -} - -// Get mocks base method. -func (m *MockSecurityGroupsClient) Get(arg0 context.Context, arg1, arg2, arg3 string) (network.SecurityGroup, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(network.SecurityGroup) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Get indicates an expected call of Get. -func (mr *MockSecurityGroupsClientMockRecorder) Get(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockSecurityGroupsClient)(nil).Get), arg0, arg1, arg2, arg3) -} - -// List mocks base method. -func (m *MockSecurityGroupsClient) List(arg0 context.Context, arg1 string) ([]network.SecurityGroup, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0, arg1) - ret0, _ := ret[0].([]network.SecurityGroup) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// List indicates an expected call of List. -func (mr *MockSecurityGroupsClientMockRecorder) List(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockSecurityGroupsClient)(nil).List), arg0, arg1) -} - -// MockVirtualNetworkPeeringsClient is a mock of VirtualNetworkPeeringsClient interface. -type MockVirtualNetworkPeeringsClient struct { - ctrl *gomock.Controller - recorder *MockVirtualNetworkPeeringsClientMockRecorder -} - -// MockVirtualNetworkPeeringsClientMockRecorder is the mock recorder for MockVirtualNetworkPeeringsClient. -type MockVirtualNetworkPeeringsClientMockRecorder struct { - mock *MockVirtualNetworkPeeringsClient -} - -// NewMockVirtualNetworkPeeringsClient creates a new mock instance. -func NewMockVirtualNetworkPeeringsClient(ctrl *gomock.Controller) *MockVirtualNetworkPeeringsClient { - mock := &MockVirtualNetworkPeeringsClient{ctrl: ctrl} - mock.recorder = &MockVirtualNetworkPeeringsClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockVirtualNetworkPeeringsClient) EXPECT() *MockVirtualNetworkPeeringsClientMockRecorder { - return m.recorder -} - -// CreateOrUpdate mocks base method. -func (m *MockVirtualNetworkPeeringsClient) CreateOrUpdate(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.VirtualNetworkPeering) (network.VirtualNetworkPeeringsCreateOrUpdateFuture, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateOrUpdate", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(network.VirtualNetworkPeeringsCreateOrUpdateFuture) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateOrUpdate indicates an expected call of CreateOrUpdate. -func (mr *MockVirtualNetworkPeeringsClientMockRecorder) CreateOrUpdate(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).CreateOrUpdate), arg0, arg1, arg2, arg3, arg4) -} - -// CreateOrUpdateAndWait mocks base method. -func (m *MockVirtualNetworkPeeringsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.VirtualNetworkPeering) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait. -func (mr *MockVirtualNetworkPeeringsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3, arg4) -} - -// Delete mocks base method. -func (m *MockVirtualNetworkPeeringsClient) Delete(arg0 context.Context, arg1, arg2, arg3 string) (network.VirtualNetworkPeeringsDeleteFuture, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(network.VirtualNetworkPeeringsDeleteFuture) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Delete indicates an expected call of Delete. -func (mr *MockVirtualNetworkPeeringsClientMockRecorder) Delete(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).Delete), arg0, arg1, arg2, arg3) -} - -// DeleteAndWait mocks base method. -func (m *MockVirtualNetworkPeeringsClient) DeleteAndWait(arg0 context.Context, arg1, arg2, arg3 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAndWait indicates an expected call of DeleteAndWait. -func (mr *MockVirtualNetworkPeeringsClientMockRecorder) DeleteAndWait(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockVirtualNetworkPeeringsClient)(nil).DeleteAndWait), arg0, arg1, arg2, arg3) -} - // MockUsageClient is a mock of UsageClient interface. type MockUsageClient struct { ctrl *gomock.Controller