diff --git a/pkg/cluster/deploybaseresources_additional.go b/pkg/cluster/deploybaseresources_additional.go index d9e8b0bbe29..7a51b6723fc 100644 --- a/pkg/cluster/deploybaseresources_additional.go +++ b/pkg/cluster/deploybaseresources_additional.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/azureclient" + "github.com/Azure/ARO-RP/pkg/util/loadbalancer" "github.com/Azure/ARO-RP/pkg/util/rbac" "github.com/Azure/ARO-RP/pkg/util/stringutils" ) @@ -531,7 +532,7 @@ func (m *manager) networkPublicLoadBalancer(azureRegion string, outboundIPs []ap resourceGroupID := m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID frontendIPConfigName := stringutils.LastTokenByte(outboundIPs[i].ID, '/') frontendConfigID := fmt.Sprintf("%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", resourceGroupID, *lb.Name, frontendIPConfigName) - *lb.FrontendIPConfigurations = append(*lb.FrontendIPConfigurations, newFrontendIPConfig(frontendIPConfigName, frontendConfigID, outboundIPs[i].ID)) + *lb.FrontendIPConfigurations = append(*lb.FrontendIPConfigurations, loadbalancer.NewFrontendIPConfig(frontendIPConfigName, frontendConfigID, outboundIPs[i].ID)) } for i := 0; i < m.doc.OpenShiftCluster.Properties.NetworkProfile.LoadBalancerProfile.ManagedOutboundIPs.Count; i++ { @@ -539,12 +540,12 @@ func (m *manager) networkPublicLoadBalancer(azureRegion string, outboundIPs []ap if i == 0 && m.doc.OpenShiftCluster.Properties.APIServerProfile.Visibility == api.VisibilityPublic { frontendIPConfigName := "public-lb-ip-v4" frontendConfigID := fmt.Sprintf("%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", resourceGroupID, *lb.Name, frontendIPConfigName) - *(*lb.OutboundRules)[0].FrontendIPConfigurations = append(*(*lb.OutboundRules)[0].FrontendIPConfigurations, newOutboundRuleFrontendIPConfig(frontendConfigID)) + *(*lb.OutboundRules)[0].FrontendIPConfigurations = append(*(*lb.OutboundRules)[0].FrontendIPConfigurations, loadbalancer.NewOutboundRuleFrontendIPConfig(frontendConfigID)) continue } frontendIPConfigName := stringutils.LastTokenByte(outboundIPs[i].ID, '/') frontendConfigID := fmt.Sprintf("%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", resourceGroupID, *lb.Name, frontendIPConfigName) - *(*lb.OutboundRules)[0].FrontendIPConfigurations = append(*(*lb.OutboundRules)[0].FrontendIPConfigurations, newOutboundRuleFrontendIPConfig(frontendConfigID)) + *(*lb.OutboundRules)[0].FrontendIPConfigurations = append(*(*lb.OutboundRules)[0].FrontendIPConfigurations, loadbalancer.NewOutboundRuleFrontendIPConfig(frontendConfigID)) } } diff --git a/pkg/cluster/loadbalancerprofile.go b/pkg/cluster/loadbalancerprofile.go index 112c5ea9470..993a4d91143 100644 --- a/pkg/cluster/loadbalancerprofile.go +++ b/pkg/cluster/loadbalancerprofile.go @@ -13,12 +13,11 @@ import ( mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/loadbalancer" "github.com/Azure/ARO-RP/pkg/util/stringutils" "github.com/Azure/ARO-RP/pkg/util/uuid" ) -const outboundRuleV4 = "outbound-rule-v4" - func (m *manager) reconcileLoadBalancerProfile(ctx context.Context) error { if m.doc.OpenShiftCluster.Properties.NetworkProfile.OutboundType != api.OutboundTypeLoadbalancer || m.doc.OpenShiftCluster.Properties.ArchitectureVersion == api.ArchitectureVersionV1 { return nil @@ -59,7 +58,7 @@ func (m *manager) reconcileOutboundRuleV4IPsInner(ctx context.Context, lb mgmtne resourceGroupName := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') infraID := m.doc.OpenShiftCluster.Properties.InfraID - originalOutboundIPs := getOutboundIPsFromLB(lb) + originalOutboundIPs := loadbalancer.GetOutboundIPsFromLB(lb) if needsEffectiveOutboundIPsPatched(m.doc.OpenShiftCluster.Properties.NetworkProfile.LoadBalancerProfile.EffectiveOutboundIPs, originalOutboundIPs) { err := m.patchEffectiveOutboundIPs(ctx, originalOutboundIPs) @@ -78,8 +77,8 @@ func (m *manager) reconcileOutboundRuleV4IPsInner(ctx context.Context, lb mgmtne } // rebuild outbound-rule-v4 frontend ip config with desired outbound ips - removeOutboundIPsFromLB(lb) - addOutboundIPsToLB(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, lb, desiredOutboundIPs) + loadbalancer.RemoveOutboundIPsFromLB(lb) + loadbalancer.AddOutboundIPsToLB(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, lb, desiredOutboundIPs) err = m.loadBalancers.CreateOrUpdateAndWait(ctx, resourceGroupName, infraID, lb) if err != nil { @@ -95,90 +94,6 @@ func (m *manager) reconcileOutboundRuleV4IPsInner(ctx context.Context, lb mgmtne return nil } -// Remove outbound-rule-v4 IPs and corresponding frontendIPConfig from load balancer -func removeOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) { - removeOutboundRuleV4FrontendIPConfig(lb) - setOutboundRuleV4(lb, []mgmtnetwork.SubResource{}) -} - -func removeOutboundRuleV4FrontendIPConfig(lb mgmtnetwork.LoadBalancer) { - var savedFIPConfig = make([]mgmtnetwork.FrontendIPConfiguration, 0, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) - var outboundRuleFrontendConfig = getOutboundRuleV4FIPConfigs(lb) - - for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { - fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID - fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] - hasLBRules := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].LoadBalancingRules != nil - if _, ok := outboundRuleFrontendConfig[fipConfigID]; ok && !hasLBRules { - continue - } - savedFIPConfig = append(savedFIPConfig, fipConfig) - } - lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &savedFIPConfig -} - -func getOutboundRuleV4FIPConfigs(lb mgmtnetwork.LoadBalancer) map[string]mgmtnetwork.SubResource { - var obRuleV4FIPConfigs = make(map[string]mgmtnetwork.SubResource) - for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { - if *obRule.Name == outboundRuleV4 { - for i := 0; i < len(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations); i++ { - fipConfigID := *(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i].ID - fipConfig := (*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i] - obRuleV4FIPConfigs[fipConfigID] = fipConfig - } - break - } - } - return obRuleV4FIPConfigs -} - -// Returns a map of Frontend IP Configurations. Frontend IP Configurations can be looked up by Public IP Address ID or Frontend IP Configuration ID -func getFrontendIPConfigs(lb mgmtnetwork.LoadBalancer) map[string]mgmtnetwork.FrontendIPConfiguration { - var frontendIPConfigs = make(map[string]mgmtnetwork.FrontendIPConfiguration, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) - - for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { - fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID - fipConfigIPAddressID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].FrontendIPConfigurationPropertiesFormat.PublicIPAddress.ID - fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] - frontendIPConfigs[fipConfigID] = fipConfig - frontendIPConfigs[fipConfigIPAddressID] = fipConfig - } - - return frontendIPConfigs -} - -// Adds IPs or IPPrefixes to the load balancer outbound rule "outbound-rule-v4". -func addOutboundIPsToLB(resourceGroupID string, lb mgmtnetwork.LoadBalancer, obIPsOrIPPrefixes []api.ResourceReference) { - frontendIPConfigs := getFrontendIPConfigs(lb) - outboundRuleV4FrontendIPConfig := []mgmtnetwork.SubResource{} - - // add IP Addresses to frontendConfig - for _, obIPOrIPPrefix := range obIPsOrIPPrefixes { - // check if the frontend config exists in the map to avoid duplicate entries - if _, ok := frontendIPConfigs[obIPOrIPPrefix.ID]; !ok { - frontendIPConfigName := stringutils.LastTokenByte(obIPOrIPPrefix.ID, '/') - frontendConfigID := fmt.Sprintf("%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", resourceGroupID, *lb.Name, frontendIPConfigName) - *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = append(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations, newFrontendIPConfig(frontendIPConfigName, frontendConfigID, obIPOrIPPrefix.ID)) - outboundRuleV4FrontendIPConfig = append(outboundRuleV4FrontendIPConfig, newOutboundRuleFrontendIPConfig(frontendConfigID)) - } else { - // frontendIPConfig already exists and just needs to be added to the outbound rule - frontendConfig := frontendIPConfigs[obIPOrIPPrefix.ID] - outboundRuleV4FrontendIPConfig = append(outboundRuleV4FrontendIPConfig, newOutboundRuleFrontendIPConfig(*frontendConfig.ID)) - } - } - - setOutboundRuleV4(lb, outboundRuleV4FrontendIPConfig) -} - -func setOutboundRuleV4(lb mgmtnetwork.LoadBalancer, outboundRuleV4FrontendIPConfig []mgmtnetwork.SubResource) { - for _, outboundRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { - if *outboundRule.Name == outboundRuleV4 { - outboundRule.OutboundRulePropertiesFormat.FrontendIPConfigurations = &outboundRuleV4FrontendIPConfig - break - } - } -} - type deleteResult struct { name string err error @@ -240,7 +155,7 @@ func (m *manager) getManagedIPsToDelete(ctx context.Context) ([]string, error) { return nil, err } - outboundIPs := getOutboundIPsFromLB(lb) + outboundIPs := loadbalancer.GetOutboundIPsFromLB(lb) outboundIPMap := make(map[string]api.ResourceReference, len(outboundIPs)) for i := 0; i < len(outboundIPs); i++ { outboundIPMap[strings.ToLower(outboundIPs[i].ID)] = outboundIPs[i] @@ -384,24 +299,6 @@ func (m *manager) createPublicIPAddress(ctx context.Context, ch chan<- createIPR } } -func getOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) []api.ResourceReference { - var outboundIPs []api.ResourceReference - fipConfigs := getFrontendIPConfigs(lb) - - for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { - if *obRule.Name == outboundRuleV4 { - for i := 0; i < len(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations); i++ { - id := *(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i].ID - if fipConfig, ok := fipConfigs[id]; ok { - outboundIPs = append(outboundIPs, api.ResourceReference{ID: *fipConfig.PublicIPAddress.ID}) - } - } - } - } - - return outboundIPs -} - func (m *manager) patchEffectiveOutboundIPs(ctx context.Context, outboundIPs []api.ResourceReference) error { m.log.Info("patching effectiveOutboundIPs") effectiveOutboundIPs := make([]api.EffectiveOutboundIP, 0, len(outboundIPs)) @@ -434,25 +331,6 @@ func newPublicIPAddress(name, resourceID, location string) mgmtnetwork.PublicIPA } } -func newFrontendIPConfig(name string, id string, publicIPorIPPrefixID string) mgmtnetwork.FrontendIPConfiguration { - // TODO: add check for publicIPorIPPrefixID - return mgmtnetwork.FrontendIPConfiguration{ - Name: &name, - ID: &id, - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: &publicIPorIPPrefixID, - }, - }, - } -} - -func newOutboundRuleFrontendIPConfig(id string) mgmtnetwork.SubResource { - return mgmtnetwork.SubResource{ - ID: &id, - } -} - func needsEffectiveOutboundIPsPatched(cosmosEffectiveOutboundIPs []api.EffectiveOutboundIP, lbEffectiveIPs []api.ResourceReference) bool { if cosmosEffectiveOutboundIPs == nil { return true diff --git a/pkg/cluster/loadbalancerprofile_test.go b/pkg/cluster/loadbalancerprofile_test.go index 1e2d21e55b9..f44970bbfbb 100644 --- a/pkg/cluster/loadbalancerprofile_test.go +++ b/pkg/cluster/loadbalancerprofile_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/loadbalancer" mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/uuid" uuidfake "github.com/Azure/ARO-RP/pkg/util/uuid/fake" @@ -182,7 +183,7 @@ func TestDeleteUnusedManagedIPs(t *testing.T) { Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT().DeleteAndWait(gomock.Any(), "clusterRG", "uuid1-outbound-pip-v4") }, expectedErr: nil, @@ -247,7 +248,7 @@ func TestDeleteUnusedManagedIPs(t *testing.T) { }, OutboundRules: &[]mgmtnetwork.OutboundRule{ { - Name: to.StringPtr(outboundRuleV4), + Name: to.StringPtr(loadbalancer.OutboundRuleV4), OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ { @@ -286,269 +287,6 @@ func TestDeleteUnusedManagedIPs(t *testing.T) { } } -func TestAddOutboundIPsToLB(t *testing.T) { - clusterRGID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG" - - // Run tests - for _, tt := range []struct { - name string - desiredOBIPs []api.ResourceReference - currentLB mgmtnetwork.LoadBalancer - expectedLB mgmtnetwork.LoadBalancer - }{ - { - name: "add default IP to lb", - desiredOBIPs: []api.ResourceReference{ - { - ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4", - }, - }, - currentLB: getClearedLB(), - expectedLB: mgmtnetwork.LoadBalancer{ - Name: to.StringPtr("infraID"), - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), - }, - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), - }, - }, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), - }, - }, - }, - { - Name: to.StringPtr("public-lb-ip-v4"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("api-internal-v4"), - }, - }, - OutboundRules: &[]mgmtnetwork.SubResource{{ - ID: to.StringPtr(outboundRuleV4), - }}, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), - }, - }, - }, - }, - OutboundRules: &[]mgmtnetwork.OutboundRule{ - { - Name: to.StringPtr(outboundRuleV4), - OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "add multiple outbound IPs to LB", - desiredOBIPs: []api.ResourceReference{ - { - ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4", - }, - { - ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid1-outbound-pip-v4", - }, - }, - currentLB: getClearedLB(), - expectedLB: mgmtnetwork.LoadBalancer{ - Name: to.StringPtr("infraID"), - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), - }, - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), - }, - }, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), - }, - }, - }, - { - Name: to.StringPtr("public-lb-ip-v4"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("api-internal-v4"), - }, - }, - OutboundRules: &[]mgmtnetwork.SubResource{{ - ID: to.StringPtr(outboundRuleV4), - }}, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), - }, - }, - }, - { - Name: to.StringPtr("uuid1-outbound-pip-v4"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid1-outbound-pip-v4"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid1-outbound-pip-v4"), - }, - }, - }, - }, - OutboundRules: &[]mgmtnetwork.OutboundRule{ - { - Name: to.StringPtr(outboundRuleV4), - OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - }, - { - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid1-outbound-pip-v4"), - }, - }, - }, - }, - }, - }, - }, - }, - } { - t.Run(tt.name, func(t *testing.T) { - // Run addOutboundIPsToLB and assert the correct results - addOutboundIPsToLB(clusterRGID, tt.currentLB, tt.desiredOBIPs) - assert.Equal(t, tt.expectedLB, tt.currentLB) - }) - } -} - -func TestRemoveOutboundIPsFromLB(t *testing.T) { - // Run tests - for _, tt := range []struct { - name string - currentLB mgmtnetwork.LoadBalancer - expectedLB mgmtnetwork.LoadBalancer - }{ - { - name: "remove all outbound-rule-v4 fip config except api server", - currentLB: fakeLoadBalancersGet(1, api.VisibilityPublic), - expectedLB: mgmtnetwork.LoadBalancer{ - Name: to.StringPtr("infraID"), - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), - }, - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), - }, - }, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), - }, - }, - }, - { - Name: to.StringPtr("public-lb-ip-v4"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("api-internal-v4"), - }, - }, - OutboundRules: &[]mgmtnetwork.SubResource{{ - ID: to.StringPtr(outboundRuleV4), - }}, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), - }, - }, - }, - }, - OutboundRules: &[]mgmtnetwork.OutboundRule{ - { - Name: to.StringPtr(outboundRuleV4), - OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.SubResource{}, - }, - }, - }, - }, - }, - }, - { - name: "remove all outbound-rule-v4 fip config", - currentLB: fakeLoadBalancersGet(1, api.VisibilityPrivate), - expectedLB: mgmtnetwork.LoadBalancer{ - Name: to.StringPtr("infraID"), - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), - }, - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), - }, - }, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), - }, - }, - }, - }, - OutboundRules: &[]mgmtnetwork.OutboundRule{ - { - Name: to.StringPtr(outboundRuleV4), - OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.SubResource{}, - }, - }, - }, - }, - }, - }, - } { - t.Run(tt.name, func(t *testing.T) { - // Run removeOutboundIPsFromLB and assert correct results - removeOutboundIPsFromLB(tt.currentLB) - assert.Equal(t, tt.expectedLB, tt.currentLB) - }) - } -} - func TestReconcileLoadBalancerProfile(t *testing.T) { ctx := context.Background() infraID := "infraID" @@ -673,13 +411,13 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { Return(getFakePublicIPList(0), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(0), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) }, expectedLoadBalancerProfile: &api.LoadBalancerProfile{ ManagedOutboundIPs: &api.ManagedOutboundIPs{ @@ -738,13 +476,13 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(1, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(1, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(1, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(1, api.VisibilityPublic), nil) }, expectedLoadBalancerProfile: &api.LoadBalancerProfile{ ManagedOutboundIPs: &api.ManagedOutboundIPs{ @@ -808,15 +546,15 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { CreateOrUpdateAndWait(ctx, clusterRGName, "uuid1-outbound-pip-v4", getFakePublicIPAddress("uuid1-outbound-pip-v4", location)).Return(nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) loadBalancersClient.EXPECT(). - CreateOrUpdateAndWait(ctx, clusterRGName, infraID, fakeUpdatedLoadBalancer(1)).Return(nil) + CreateOrUpdateAndWait(ctx, clusterRGName, infraID, loadbalancer.FakeUpdatedLoadBalancer(1)).Return(nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(1, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(1, api.VisibilityPublic), nil) }, expectedLoadBalancerProfile: &api.LoadBalancerProfile{ ManagedOutboundIPs: &api.ManagedOutboundIPs{ @@ -882,15 +620,15 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(1, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(1, api.VisibilityPublic), nil) loadBalancersClient.EXPECT(). - CreateOrUpdateAndWait(ctx, clusterRGName, infraID, fakeUpdatedLoadBalancer(0)).Return(nil) + CreateOrUpdateAndWait(ctx, clusterRGName, infraID, loadbalancer.FakeUpdatedLoadBalancer(0)).Return(nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT().DeleteAndWait(gomock.Any(), "clusterRG", "uuid1-outbound-pip-v4") }, @@ -953,15 +691,15 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { CreateOrUpdateAndWait(ctx, clusterRGName, "uuid1-outbound-pip-v4", getFakePublicIPAddress("uuid1-outbound-pip-v4", location)).Return(nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) loadBalancersClient.EXPECT(). - CreateOrUpdateAndWait(ctx, clusterRGName, infraID, fakeUpdatedLoadBalancer(1)).Return(fmt.Errorf("lb update failed")) + CreateOrUpdateAndWait(ctx, clusterRGName, infraID, loadbalancer.FakeUpdatedLoadBalancer(1)).Return(fmt.Errorf("lb update failed")) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT().DeleteAndWait(gomock.Any(), "clusterRG", "uuid1-outbound-pip-v4") }, expectedLoadBalancerProfile: &api.LoadBalancerProfile{ @@ -1027,15 +765,15 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { Return(getFakePublicIPList(2), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(2, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(2, api.VisibilityPublic), nil) loadBalancersClient.EXPECT(). - CreateOrUpdateAndWait(ctx, clusterRGName, infraID, fakeUpdatedLoadBalancer(0)).Return(nil) + CreateOrUpdateAndWait(ctx, clusterRGName, infraID, loadbalancer.FakeUpdatedLoadBalancer(0)).Return(nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(2), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT().DeleteAndWait(gomock.Any(), "clusterRG", "uuid1-outbound-pip-v4").Return(fmt.Errorf("error")) publicIPAddressClient.EXPECT().DeleteAndWait(gomock.Any(), "clusterRG", "uuid2-outbound-pip-v4").Return(fmt.Errorf("error")) }, @@ -1100,13 +838,13 @@ func TestReconcileLoadBalancerProfile(t *testing.T) { CreateOrUpdateAndWait(ctx, clusterRGName, "uuid2-outbound-pip-v4", getFakePublicIPAddress("uuid2-outbound-pip-v4", location)).Return(fmt.Errorf("failed to create ip")) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT(). List(gomock.Any(), clusterRGName). Return(getFakePublicIPList(1), nil) loadBalancersClient.EXPECT(). Get(gomock.Any(), clusterRGName, infraID, ""). - Return(fakeLoadBalancersGet(0, api.VisibilityPublic), nil) + Return(loadbalancer.FakeLoadBalancersGet(0, api.VisibilityPublic), nil) publicIPAddressClient.EXPECT().DeleteAndWait(gomock.Any(), "clusterRG", "uuid1-outbound-pip-v4").Return(fmt.Errorf("error")) }, expectedLoadBalancerProfile: &api.LoadBalancerProfile{ @@ -1175,110 +913,6 @@ func getFakePublicIPAddress(name, location string) mgmtnetwork.PublicIPAddress { } } -// Returns a load balancer with config updated with desired outbound ips as it should be when m.loadBalancersClient.CreateOrUpdate is called. -// It is assumed that desired IPs include the default outbound IPs, however this won't work for transitions from -// customer provided IPs/Prefixes to managed IPs if the api server is private since the default IP -// would be deleted -func fakeUpdatedLoadBalancer(additionalIPCount int) mgmtnetwork.LoadBalancer { - clusterRGID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG" - defaultOutboundIPID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4" - lb := getClearedLB() - ipResourceRefs := []api.ResourceReference{} - ipResourceRefs = append(ipResourceRefs, api.ResourceReference{ID: defaultOutboundIPID}) - for i := 0; i < additionalIPCount; i++ { - ipResourceRefs = append(ipResourceRefs, api.ResourceReference{ID: fmt.Sprintf("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid%d-outbound-pip-v4", i+1)}) - } - addOutboundIPsToLB(clusterRGID, lb, ipResourceRefs) - return lb -} - -// Returns lb as it would be returned via m.loadBalancersClient.Get. -func fakeLoadBalancersGet(additionalIPCount int, apiServerVisibility api.Visibility) mgmtnetwork.LoadBalancer { - defaultOutboundFIPConfig := mgmtnetwork.FrontendIPConfiguration{ - Name: to.StringPtr("public-lb-ip-v4"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - OutboundRules: &[]mgmtnetwork.SubResource{{ - ID: to.StringPtr(outboundRuleV4), - }}, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), - }, - }, - } - if apiServerVisibility == api.VisibilityPublic { - defaultOutboundFIPConfig.FrontendIPConfigurationPropertiesFormat.LoadBalancingRules = &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("api-internal-v4"), - }, - } - } - lb := mgmtnetwork.LoadBalancer{ - Name: to.StringPtr("infraID"), - LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ - { - Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - LoadBalancingRules: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), - }, - { - ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), - }, - }, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), - }, - }, - }, - defaultOutboundFIPConfig, - }, - OutboundRules: &[]mgmtnetwork.OutboundRule{ - { - Name: to.StringPtr(outboundRuleV4), - OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ - { - ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), - }, - }, - }, - }, - }, - }, - } - for i := 0; i < additionalIPCount; i++ { - fipName := fmt.Sprintf("uuid%d-outbound-pip-v4", i+1) - ipID := fmt.Sprintf("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid%d-outbound-pip-v4", i+1) - fipID := fmt.Sprintf("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid%d-outbound-pip-v4", i+1) - fipConfig := mgmtnetwork.FrontendIPConfiguration{ - Name: &fipName, - ID: &fipID, - FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ - OutboundRules: &[]mgmtnetwork.SubResource{{ - ID: to.StringPtr(outboundRuleV4), - }}, - PublicIPAddress: &mgmtnetwork.PublicIPAddress{ - ID: &ipID, - }, - }, - } - *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = append(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations, fipConfig) - outboundRules := *lb.LoadBalancerPropertiesFormat.OutboundRules - *outboundRules[0].FrontendIPConfigurations = append(*outboundRules[0].FrontendIPConfigurations, mgmtnetwork.SubResource{ID: fipConfig.ID}) - } - return lb -} - -func getClearedLB() mgmtnetwork.LoadBalancer { - lb := fakeLoadBalancersGet(0, api.VisibilityPublic) - removeOutboundIPsFromLB(lb) - return lb -} - func getFakePublicIPList(managedCount int) []mgmtnetwork.PublicIPAddress { infraID := "infraID" clusterRGID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG" diff --git a/pkg/util/loadbalancer/loadbalancer.go b/pkg/util/loadbalancer/loadbalancer.go index cefdd518cce..d18b862a0c0 100644 --- a/pkg/util/loadbalancer/loadbalancer.go +++ b/pkg/util/loadbalancer/loadbalancer.go @@ -8,6 +8,10 @@ import ( "strings" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/go-autorest/autorest/to" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/stringutils" ) func RemoveFrontendIPConfiguration(lb *mgmtnetwork.LoadBalancer, resourceID string) error { @@ -28,3 +32,232 @@ func RemoveFrontendIPConfiguration(lb *mgmtnetwork.LoadBalancer, resourceID stri func isFrontendIPConfigReferenced(fipConfig mgmtnetwork.FrontendIPConfiguration) bool { return fipConfig.LoadBalancingRules != nil || fipConfig.InboundNatPools != nil || fipConfig.InboundNatRules != nil || fipConfig.OutboundRules != nil } + +const OutboundRuleV4 = "outbound-rule-v4" + +// Remove outbound-rule-v4 IPs and corresponding frontendIPConfig from load balancer +func RemoveOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) { + removeOutboundRuleV4FrontendIPConfig(lb) + setOutboundRuleV4(lb, []mgmtnetwork.SubResource{}) +} + +func removeOutboundRuleV4FrontendIPConfig(lb mgmtnetwork.LoadBalancer) { + var savedFIPConfig = make([]mgmtnetwork.FrontendIPConfiguration, 0, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) + var outboundRuleFrontendConfig = getOutboundRuleV4FIPConfigs(lb) + + for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { + fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID + fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] + hasLBRules := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].LoadBalancingRules != nil + if _, ok := outboundRuleFrontendConfig[fipConfigID]; ok && !hasLBRules { + continue + } + savedFIPConfig = append(savedFIPConfig, fipConfig) + } + lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &savedFIPConfig +} + +func getOutboundRuleV4FIPConfigs(lb mgmtnetwork.LoadBalancer) map[string]mgmtnetwork.SubResource { + var obRuleV4FIPConfigs = make(map[string]mgmtnetwork.SubResource) + for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { + if *obRule.Name == OutboundRuleV4 { + for i := 0; i < len(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations); i++ { + fipConfigID := *(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i].ID + fipConfig := (*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i] + obRuleV4FIPConfigs[fipConfigID] = fipConfig + } + break + } + } + return obRuleV4FIPConfigs +} + +// Returns a map of Frontend IP Configurations. Frontend IP Configurations can be looked up by Public IP Address ID or Frontend IP Configuration ID +func getFrontendIPConfigs(lb mgmtnetwork.LoadBalancer) map[string]mgmtnetwork.FrontendIPConfiguration { + var frontendIPConfigs = make(map[string]mgmtnetwork.FrontendIPConfiguration, len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)) + + for i := 0; i < len(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations); i++ { + fipConfigID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].ID + fipConfigIPAddressID := *(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i].FrontendIPConfigurationPropertiesFormat.PublicIPAddress.ID + fipConfig := (*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations)[i] + frontendIPConfigs[fipConfigID] = fipConfig + frontendIPConfigs[fipConfigIPAddressID] = fipConfig + } + + return frontendIPConfigs +} + +// Adds IPs or IPPrefixes to the load balancer outbound rule "outbound-rule-v4". +func AddOutboundIPsToLB(resourceGroupID string, lb mgmtnetwork.LoadBalancer, obIPsOrIPPrefixes []api.ResourceReference) { + frontendIPConfigs := getFrontendIPConfigs(lb) + outboundRuleV4FrontendIPConfig := []mgmtnetwork.SubResource{} + + // add IP Addresses to frontendConfig + for _, obIPOrIPPrefix := range obIPsOrIPPrefixes { + // check if the frontend config exists in the map to avoid duplicate entries + if _, ok := frontendIPConfigs[obIPOrIPPrefix.ID]; !ok { + frontendIPConfigName := stringutils.LastTokenByte(obIPOrIPPrefix.ID, '/') + frontendConfigID := fmt.Sprintf("%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s", resourceGroupID, *lb.Name, frontendIPConfigName) + *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = append(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations, NewFrontendIPConfig(frontendIPConfigName, frontendConfigID, obIPOrIPPrefix.ID)) + outboundRuleV4FrontendIPConfig = append(outboundRuleV4FrontendIPConfig, NewOutboundRuleFrontendIPConfig(frontendConfigID)) + } else { + // frontendIPConfig already exists and just needs to be added to the outbound rule + frontendConfig := frontendIPConfigs[obIPOrIPPrefix.ID] + outboundRuleV4FrontendIPConfig = append(outboundRuleV4FrontendIPConfig, NewOutboundRuleFrontendIPConfig(*frontendConfig.ID)) + } + } + + setOutboundRuleV4(lb, outboundRuleV4FrontendIPConfig) +} + +func GetOutboundIPsFromLB(lb mgmtnetwork.LoadBalancer) []api.ResourceReference { + var outboundIPs []api.ResourceReference + fipConfigs := getFrontendIPConfigs(lb) + + for _, obRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { + if *obRule.Name == OutboundRuleV4 { + for i := 0; i < len(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations); i++ { + id := *(*obRule.OutboundRulePropertiesFormat.FrontendIPConfigurations)[i].ID + if fipConfig, ok := fipConfigs[id]; ok { + outboundIPs = append(outboundIPs, api.ResourceReference{ID: *fipConfig.PublicIPAddress.ID}) + } + } + } + } + + return outboundIPs +} + +func setOutboundRuleV4(lb mgmtnetwork.LoadBalancer, outboundRuleV4FrontendIPConfig []mgmtnetwork.SubResource) { + for _, outboundRule := range *lb.LoadBalancerPropertiesFormat.OutboundRules { + if *outboundRule.Name == OutboundRuleV4 { + outboundRule.OutboundRulePropertiesFormat.FrontendIPConfigurations = &outboundRuleV4FrontendIPConfig + break + } + } +} + +func NewFrontendIPConfig(name string, id string, publicIPorIPPrefixID string) mgmtnetwork.FrontendIPConfiguration { + // TODO: add check for publicIPorIPPrefixID + return mgmtnetwork.FrontendIPConfiguration{ + Name: &name, + ID: &id, + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: &publicIPorIPPrefixID, + }, + }, + } +} + +func NewOutboundRuleFrontendIPConfig(id string) mgmtnetwork.SubResource { + return mgmtnetwork.SubResource{ + ID: &id, + } +} + +// The following functions are only used for testing, but must be exported to ease testing in pkg/cluster/loadbalancerprofile_test.go + +// Returns a load balancer with config updated with desired outbound ips as it should be when m.loadBalancersClient.CreateOrUpdate is called. +// It is assumed that desired IPs include the default outbound IPs, however this won't work for transitions from +// customer provided IPs/Prefixes to managed IPs if the api server is private since the default IP +// would be deleted +func FakeUpdatedLoadBalancer(additionalIPCount int) mgmtnetwork.LoadBalancer { + clusterRGID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG" + defaultOutboundIPID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4" + lb := GetClearedLB() + ipResourceRefs := []api.ResourceReference{} + ipResourceRefs = append(ipResourceRefs, api.ResourceReference{ID: defaultOutboundIPID}) + for i := 0; i < additionalIPCount; i++ { + ipResourceRefs = append(ipResourceRefs, api.ResourceReference{ID: fmt.Sprintf("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid%d-outbound-pip-v4", i+1)}) + } + AddOutboundIPsToLB(clusterRGID, lb, ipResourceRefs) + return lb +} + +// Returns lb as it would be returned via m.loadBalancersClient.Get. +func FakeLoadBalancersGet(additionalIPCount int, apiServerVisibility api.Visibility) mgmtnetwork.LoadBalancer { + defaultOutboundFIPConfig := mgmtnetwork.FrontendIPConfiguration{ + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(OutboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + } + if apiServerVisibility == api.VisibilityPublic { + defaultOutboundFIPConfig.FrontendIPConfigurationPropertiesFormat.LoadBalancingRules = &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + } + } + lb := mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + defaultOutboundFIPConfig, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(OutboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + }, + }, + }, + }, + }, + }, + } + for i := 0; i < additionalIPCount; i++ { + fipName := fmt.Sprintf("uuid%d-outbound-pip-v4", i+1) + ipID := fmt.Sprintf("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid%d-outbound-pip-v4", i+1) + fipID := fmt.Sprintf("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid%d-outbound-pip-v4", i+1) + fipConfig := mgmtnetwork.FrontendIPConfiguration{ + Name: &fipName, + ID: &fipID, + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(OutboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: &ipID, + }, + }, + } + *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = append(*lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations, fipConfig) + outboundRules := *lb.LoadBalancerPropertiesFormat.OutboundRules + *outboundRules[0].FrontendIPConfigurations = append(*outboundRules[0].FrontendIPConfigurations, mgmtnetwork.SubResource{ID: fipConfig.ID}) + } + return lb +} + +func GetClearedLB() mgmtnetwork.LoadBalancer { + lb := FakeLoadBalancersGet(0, api.VisibilityPublic) + RemoveOutboundIPsFromLB(lb) + return lb +} diff --git a/pkg/util/loadbalancer/loadbalancer_test.go b/pkg/util/loadbalancer/loadbalancer_test.go index 6b02d0106dd..c6deda3a61b 100644 --- a/pkg/util/loadbalancer/loadbalancer_test.go +++ b/pkg/util/loadbalancer/loadbalancer_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/stretchr/testify/assert" + "github.com/Azure/ARO-RP/pkg/api" utilerror "github.com/Azure/ARO-RP/test/util/error" ) @@ -132,3 +133,343 @@ func TestRemoveLoadBalancerFrontendIPConfiguration(t *testing.T) { }) } } + +func TestGetOutboundIPsFromLB(t *testing.T) { + // Run tests + for _, tt := range []struct { + name string + currentLB mgmtnetwork.LoadBalancer + expectedOutboundIPs []api.ResourceReference + }{ + { + name: "default", + currentLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + }, + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(OutboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(OutboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + }, + }, + }, + }, + }, + }, + }, + expectedOutboundIPs: []api.ResourceReference{ + { + ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4", + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Run GetOutboundIPsFromLB and assert the correct results + outboundIPs := GetOutboundIPsFromLB(tt.currentLB) + assert.Equal(t, tt.expectedOutboundIPs, outboundIPs) + }) + } +} + +func TestAddOutboundIPsToLB(t *testing.T) { + clusterRGID := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG" + + // Run tests + for _, tt := range []struct { + name string + desiredOBIPs []api.ResourceReference + currentLB mgmtnetwork.LoadBalancer + expectedLB mgmtnetwork.LoadBalancer + }{ + { + name: "add default IP to lb", + desiredOBIPs: []api.ResourceReference{ + { + ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4", + }, + }, + currentLB: GetClearedLB(), + expectedLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + }, + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(OutboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(OutboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "add multiple outbound IPs to LB", + desiredOBIPs: []api.ResourceReference{ + { + ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4", + }, + { + ID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid1-outbound-pip-v4", + }, + }, + currentLB: GetClearedLB(), + expectedLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + }, + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(OutboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + }, + { + Name: to.StringPtr("uuid1-outbound-pip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid1-outbound-pip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/uuid1-outbound-pip-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(OutboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + }, + { + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/uuid1-outbound-pip-v4"), + }, + }, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Run addOutboundIPsToLB and assert the correct results + AddOutboundIPsToLB(clusterRGID, tt.currentLB, tt.desiredOBIPs) + assert.Equal(t, tt.expectedLB, tt.currentLB) + }) + } +} + +func TestRemoveOutboundIPsFromLB(t *testing.T) { + // Run tests + for _, tt := range []struct { + name string + currentLB mgmtnetwork.LoadBalancer + expectedLB mgmtnetwork.LoadBalancer + }{ + { + name: "remove all outbound-rule-v4 fip config except api server", + currentLB: FakeLoadBalancersGet(1, api.VisibilityPublic), + expectedLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("public-lb-ip-v4"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("api-internal-v4"), + }, + }, + OutboundRules: &[]mgmtnetwork.SubResource{{ + ID: to.StringPtr(OutboundRuleV4), + }}, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(OutboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{}, + }, + }, + }, + }, + }, + }, + { + name: "remove all outbound-rule-v4 fip config", + currentLB: FakeLoadBalancersGet(1, api.VisibilityPrivate), + expectedLB: mgmtnetwork.LoadBalancer{ + Name: to.StringPtr("infraID"), + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + }, + OutboundRules: &[]mgmtnetwork.OutboundRule{ + { + Name: to.StringPtr(OutboundRuleV4), + OutboundRulePropertiesFormat: &mgmtnetwork.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.SubResource{}, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Run removeOutboundIPsFromLB and assert correct results + RemoveOutboundIPsFromLB(tt.currentLB) + assert.Equal(t, tt.expectedLB, tt.currentLB) + }) + } +}