From 9b92b4f79b70491e8f2ae430031db7407b0c5b7c Mon Sep 17 00:00:00 2001 From: Tony Schneider Date: Wed, 29 Nov 2023 16:09:56 -0600 Subject: [PATCH] Admin action to delete a cluster managed resource (#3286) * add ResourceDeleteAndWait to azureactions * add delete resource admin action and frontend routing * add helper functions for lb config manipulation * refactor azure actions - moves resource delete code to seperate file - adds loadbalancer client to handle deleting FrontendIPConfiguration - updates ResourceDeleteAndWait to handle deleting FrontendIPConfigurations - adds DeleteByIDAndWait to features/resources client * add e2e tests * fix imports and add license headers * cleanup / fix lint * add command example to doc * rename to "managed" resource id * change query param to camel case * use var group instead * return error as adminReply already wraps in CloudError * fix missed camelCase of query param * use regex to match frontend ip configurations * remove focus * add deny list to prevent deleting PLS and Storage * fix mixed import * use fake pls name to prevent accidently deleting e2e cluster pls * fix test * add PE to deny list --- docs/deploy-development-rp.md | 6 + ...openshiftcluster_delete_managedresource.go | 67 ++++++ ...hiftcluster_delete_managedresource_test.go | 138 ++++++++++++ pkg/frontend/adminactions/azureactions.go | 3 + .../adminactions/delete_managedresource.go | 73 +++++++ .../delete_managedresource_test.go | 199 ++++++++++++++++++ pkg/frontend/frontend.go | 1 + .../mgmt/features/resources_addons.go | 10 + pkg/util/loadbalancer/loadbalancer.go | 30 +++ pkg/util/loadbalancer/loadbalancer_test.go | 134 ++++++++++++ pkg/util/mocks/adminactions/adminactions.go | 14 ++ .../azureclient/mgmt/features/features.go | 14 ++ test/e2e/adminapi_delete_managedresource.go | 128 +++++++++++ test/e2e/update.go | 4 +- 14 files changed, 819 insertions(+), 2 deletions(-) create mode 100644 pkg/frontend/admin_openshiftcluster_delete_managedresource.go create mode 100644 pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go create mode 100644 pkg/frontend/adminactions/delete_managedresource.go create mode 100644 pkg/frontend/adminactions/delete_managedresource_test.go create mode 100644 pkg/util/loadbalancer/loadbalancer.go create mode 100644 pkg/util/loadbalancer/loadbalancer_test.go create mode 100644 test/e2e/adminapi_delete_managedresource.go diff --git a/docs/deploy-development-rp.md b/docs/deploy-development-rp.md index 3d48e38c95b..5f2a97b0b8c 100644 --- a/docs/deploy-development-rp.md +++ b/docs/deploy-development-rp.md @@ -269,6 +269,12 @@ After that, when you [create](https://github.com/Azure/ARO-RP/blob/master/docs/d curl -X PATCH -k "https://localhost:8443/admin/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER/etcdrecovery" ``` +* Delete a managed resource + ```bash + MANAGED_RESOURCEID= + curl -X POST -k "https://localhost:8443/admin/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER/deletemanagedresource?managedResourceID=$MANAGED_RESOURCEID" + ``` + ## OpenShift Version * We have a cosmos container which contains supported installable OCP versions, more information on the definition in `pkg/api/openshiftversion.go`. diff --git a/pkg/frontend/admin_openshiftcluster_delete_managedresource.go b/pkg/frontend/admin_openshiftcluster_delete_managedresource.go new file mode 100644 index 00000000000..cc88eea1aaa --- /dev/null +++ b/pkg/frontend/admin_openshiftcluster_delete_managedresource.go @@ -0,0 +1,67 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "net/http" + "path/filepath" + "strings" + + "github.com/Azure/go-autorest/autorest" + "github.com/go-chi/chi/v5" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/database/cosmosdb" + "github.com/Azure/ARO-RP/pkg/frontend/middleware" +) + +func (f *frontend) postAdminOpenShiftDeleteManagedResource(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry) + r.URL.Path = filepath.Dir(r.URL.Path) + + err := f._postAdminOpenShiftClusterDeleteManagedResource(ctx, r, log) + adminReply(log, w, nil, nil, err) +} + +func (f *frontend) _postAdminOpenShiftClusterDeleteManagedResource(ctx context.Context, r *http.Request, log *logrus.Entry) error { + resType, resName, resGroupName := chi.URLParam(r, "resourceType"), chi.URLParam(r, "resourceName"), chi.URLParam(r, "resourceGroupName") + managedResourceID := r.URL.Query().Get("managedResourceID") + resourceID := strings.TrimPrefix(r.URL.Path, "/admin") + + doc, err := f.dbOpenShiftClusters.Get(ctx, resourceID) + switch { + case cosmosdb.IsErrorStatusCode(err, http.StatusNotFound): + return api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeResourceNotFound, "", "The Resource '%s/%s' under resource group '%s' was not found.", resType, resName, resGroupName) + case err != nil: + return err + } + + if !strings.HasPrefix(strings.ToLower(managedResourceID), strings.ToLower(doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID)) { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "The resource %s is not within the cluster's managed resource group %s.", managedResourceID, doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID) + } + + subscriptionDoc, err := f.getSubscriptionDocument(ctx, doc.Key) + if err != nil { + return err + } + + a, err := f.azureActionsFactory(log, f.env, doc.OpenShiftCluster, subscriptionDoc) + if err != nil { + return err + } + + err = a.ResourceDeleteAndWait(ctx, managedResourceID) + if err != nil { + if detailedErr, ok := err.(autorest.DetailedError); ok && + detailedErr.StatusCode == http.StatusNotFound { + return api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeNotFound, "", "The resource '%s' could not be found.", managedResourceID) + } + return err + } + + return nil +} diff --git a/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go b/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go new file mode 100644 index 00000000000..4215af7ceaa --- /dev/null +++ b/pkg/frontend/admin_openshiftcluster_delete_managedresource_test.go @@ -0,0 +1,138 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/Azure/go-autorest/autorest" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/frontend/adminactions" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +func TestAdminDeleteManagedResource(t *testing.T) { + mockSubID := "00000000-0000-0000-0000-000000000000" + mockTenantID := "00000000-0000-0000-0000-000000000000" + + ctx := context.Background() + + type test struct { + name string + resourceID string + managedResourceID string + mocks func(*test, *mock_adminactions.MockAzureActions) + wantStatusCode int + wantResponse []byte + wantError string + } + + for _, tt := range []*test{ + { + name: "delete managed resource within cluster managed resourcegroup", + resourceID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + managedResourceID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083", mockSubID), + mocks: func(tt *test, a *mock_adminactions.MockAzureActions) { + a.EXPECT().ResourceDeleteAndWait(gomock.Any(), tt.managedResourceID).Return(nil) + }, + wantStatusCode: http.StatusOK, + }, + { + name: "delete managed resource not within cluster managed resourcegroup fails", + resourceID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + managedResourceID: fmt.Sprintf("/subscriptions/%s/resourceGroups/notmanagedresourcegroup/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083", mockSubID), + mocks: func(tt *test, a *mock_adminactions.MockAzureActions) { + }, + wantStatusCode: http.StatusBadRequest, + wantError: "400: InvalidParameter: : The resource /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/notmanagedresourcegroup/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083 is not within the cluster's managed resource group /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-cluster.", + }, + { + name: "delete a resource that doesn't exist fails", + resourceID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + managedResourceID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083", mockSubID), + mocks: func(tt *test, a *mock_adminactions.MockAzureActions) { + a.EXPECT().ResourceDeleteAndWait(gomock.Any(), tt.managedResourceID).Return(autorest.DetailedError{StatusCode: 404}) + }, + wantStatusCode: http.StatusNotFound, + wantError: fmt.Sprintf("404: NotFound: : The resource '%s' could not be found.", fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083", mockSubID)), + }, + { + name: "cannot delete resources in the deny list", + resourceID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + managedResourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/test-cluster/providers/Microsoft.Network/privateLinkServices/infraID", mockSubID), + mocks: func(tt *test, a *mock_adminactions.MockAzureActions) { + a.EXPECT().ResourceDeleteAndWait(gomock.Any(), tt.managedResourceID).Return(api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", + fmt.Sprintf("deletion of resource /subscriptions/%s/resourcegroups/test-cluster/providers/Microsoft.Network/privateLinkServices/infraID is forbidden", mockSubID)), + ) + }, + wantStatusCode: http.StatusBadRequest, + wantError: api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", + fmt.Sprintf("deletion of resource /subscriptions/%s/resourcegroups/test-cluster/providers/Microsoft.Network/privateLinkServices/infraID is forbidden", mockSubID)).Error(), + }, + } { + t.Run(tt.name, func(t *testing.T) { + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions() + defer ti.done() + ti.fixture.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{ + Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: testdatabase.GetResourcePath(mockSubID, "resourceName"), + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster", mockSubID), + }, + }, + }, + }) + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + err := ti.buildFixtures(nil) + if err != nil { + t.Fatal(err) + } + + a := mock_adminactions.NewMockAzureActions(ti.controller) + tt.mocks(tt, a) + + f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, &noop.Noop{}, nil, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) { + return a, nil + }, nil) + + if err != nil { + t.Fatal(err) + } + + go f.Run(ctx, nil, nil) + resp, b, err := ti.request(http.MethodPost, + fmt.Sprintf("https://server/admin%s/deletemanagedresource?managedResourceID=%s", tt.resourceID, tt.managedResourceID), + nil, nil) + if err != nil { + t.Error(err) + } + + err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse) + if err != nil { + t.Error(err) + } + }) + } +} diff --git a/pkg/frontend/adminactions/azureactions.go b/pkg/frontend/adminactions/azureactions.go index fa4417cf22f..d0061262666 100644 --- a/pkg/frontend/adminactions/azureactions.go +++ b/pkg/frontend/adminactions/azureactions.go @@ -39,6 +39,7 @@ type AzureActions interface { VMSerialConsole(ctx context.Context, w http.ResponseWriter, log *logrus.Entry, vmName string) error AppLensGetDetector(ctx context.Context, detectorId string) ([]byte, error) AppLensListDetectors(ctx context.Context) ([]byte, error) + ResourceDeleteAndWait(ctx context.Context, resourceID string) error } type azureActions struct { @@ -54,6 +55,7 @@ type azureActions struct { routeTables network.RouteTablesClient storageAccounts storage.AccountsClient networkInterfaces network.InterfacesClient + loadBalancers network.LoadBalancersClient appLens applens.AppLensClient } @@ -89,6 +91,7 @@ func NewAzureActions(log *logrus.Entry, env env.Interface, oc *api.OpenShiftClus routeTables: network.NewRouteTablesClient(env.Environment(), subscriptionDoc.ID, fpAuth), storageAccounts: storage.NewAccountsClient(env.Environment(), subscriptionDoc.ID, fpAuth), networkInterfaces: network.NewInterfacesClient(env.Environment(), subscriptionDoc.ID, fpAuth), + loadBalancers: network.NewLoadBalancersClient(env.Environment(), subscriptionDoc.ID, fpAuth), appLens: appLensClient, }, nil } diff --git a/pkg/frontend/adminactions/delete_managedresource.go b/pkg/frontend/adminactions/delete_managedresource.go new file mode 100644 index 00000000000..c4e99ef839c --- /dev/null +++ b/pkg/frontend/adminactions/delete_managedresource.go @@ -0,0 +1,73 @@ +package adminactions + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "net/http" + "regexp" + "strings" + + "github.com/Azure/go-autorest/autorest/azure" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/azureclient" + "github.com/Azure/ARO-RP/pkg/util/loadbalancer" +) + +var ( + frontendIPConfigurationPattern = `(?i)^/subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft\.Network/loadBalancers/(.+)/frontendIPConfigurations/([^/]+)$` + denyList = []string{ + `(?i)^/subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft\.Network/privateLinkServices/([^/]+)$`, + `(?i)^/subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft\.Network/privateEndpoints/([^/]+)$`, + `(?i)^/subscriptions/(.+)/resourceGroups/(.+)/providers/Microsoft\.Storage/(.+)$`, + } +) + +func (a *azureActions) ResourceDeleteAndWait(ctx context.Context, resourceID string) error { + idParts, err := azure.ParseResourceID(resourceID) + if err != nil { + return err + } + + for _, regex := range denyList { + re := regexp.MustCompile(regex) + if re.MatchString(resourceID) { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "deletion of resource %s is forbidden", resourceID) + } + } + + apiVersion := azureclient.APIVersion(strings.ToLower(idParts.Provider + "/" + idParts.ResourceType)) + + _, err = a.resources.GetByID(ctx, resourceID, apiVersion) + if err != nil { + return err + } + + re := regexp.MustCompile(frontendIPConfigurationPattern) + // FrontendIPConfiguration cannot be deleted with DeleteByIDAndWait (DELETE method is invalid on frontendIPConfiguration resourceID) + if re.MatchString(resourceID) { + return a.deleteFrontendIPConfiguration(ctx, resourceID) + } + + return a.resources.DeleteByIDAndWait(ctx, resourceID, apiVersion) +} + +func (a *azureActions) deleteFrontendIPConfiguration(ctx context.Context, resourceID string) error { + idParts := strings.Split(resourceID, "/") + rg := idParts[4] + lbName := idParts[8] + + lb, err := a.loadBalancers.Get(ctx, rg, lbName, "") + if err != nil { + return err + } + + err = loadbalancer.RemoveFrontendIPConfiguration(&lb, resourceID) + if err != nil { + return err + } + + return a.loadBalancers.CreateOrUpdateAndWait(ctx, rg, lbName, lb) +} diff --git a/pkg/frontend/adminactions/delete_managedresource_test.go b/pkg/frontend/adminactions/delete_managedresource_test.go new file mode 100644 index 00000000000..8f82955f75a --- /dev/null +++ b/pkg/frontend/adminactions/delete_managedresource_test.go @@ -0,0 +1,199 @@ +package adminactions + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "testing" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + mock_features "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/features" + mock_network "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/network" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + utilerror "github.com/Azure/ARO-RP/test/util/error" +) + +var ( + infraID = "infraID" + location = "eastus" + subscription = "00000000-0000-0000-0000-000000000000" + clusterRG = "clusterRG" +) + +var originalLB = mgmtnetwork.LoadBalancer{ + Sku: &mgmtnetwork.LoadBalancerSku{ + Name: mgmtnetwork.LoadBalancerSkuNameStandard, + }, + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + Name: to.StringPtr("public-lb-ip-v4"), + }, + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("adce98f85c7dd47c5a21263a5e39c083"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/adce98f85c7dd47c5a21263a5e39c083"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083"), + }, + }, + }, + }, + }, + Name: to.StringPtr(infraID), + Type: to.StringPtr("Microsoft.Network/loadBalancers"), + Location: to.StringPtr(location), +} + +func TestDeleteManagedResource(t *testing.T) { + // Run tests + for _, tt := range []struct { + name string + resourceID string + currentLB mgmtnetwork.LoadBalancer + expectedErr string + mocks func(*mock_features.MockResourcesClient, *mock_network.MockLoadBalancersClient) + }{ + { + name: "remove frontend ip config", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/adce98f85c7dd47c5a21263a5e39c083", + currentLB: originalLB, + expectedErr: "", + mocks: func(resources *mock_features.MockResourcesClient, loadBalancers *mock_network.MockLoadBalancersClient) { + resources.EXPECT().GetByID(gomock.Any(), "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/adce98f85c7dd47c5a21263a5e39c083", "2020-08-01").Return(mgmtfeatures.GenericResource{}, nil) + loadBalancers.EXPECT().Get(gomock.Any(), "clusterRG", "infraID", "").Return(originalLB, nil) + loadBalancers.EXPECT().CreateOrUpdateAndWait(gomock.Any(), clusterRG, infraID, mgmtnetwork.LoadBalancer{ + Sku: &mgmtnetwork.LoadBalancerSku{ + Name: mgmtnetwork.LoadBalancerSkuNameStandard, + }, + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + Name: to.StringPtr("public-lb-ip-v4"), + }, + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + }, + }, + Name: to.StringPtr(infraID), + Type: to.StringPtr("Microsoft.Network/loadBalancers"), + Location: to.StringPtr(location), + }).Return(nil) + }, + }, + { + name: "delete public IP Address", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/adce98f85c7dd47c5a21263a5e39c083", + expectedErr: "", + mocks: func(resources *mock_features.MockResourcesClient, loadBalancers *mock_network.MockLoadBalancersClient) { + resources.EXPECT().GetByID(gomock.Any(), "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/adce98f85c7dd47c5a21263a5e39c083", "2020-08-01").Return(mgmtfeatures.GenericResource{}, nil) + resources.EXPECT().DeleteByIDAndWait(gomock.Any(), "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/adce98f85c7dd47c5a21263a5e39c083", "2020-08-01").Return(nil) + }, + }, + { + name: "deletion of private link service is forbidden", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/privateLinkServices/infraID-pls", + expectedErr: api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "deletion of resource /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/privateLinkServices/infraID-pls is forbidden").Error(), + mocks: func(resources *mock_features.MockResourcesClient, loadBalancers *mock_network.MockLoadBalancersClient) { + }, + }, + { + name: "deletion of private endpoints are forbidden", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/privateEndpoints/infraID-pe", + expectedErr: api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "deletion of resource /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/privateEndpoints/infraID-pe is forbidden").Error(), + mocks: func(resources *mock_features.MockResourcesClient, loadBalancers *mock_network.MockLoadBalancersClient) { + }, + }, + { + name: "deletion of Microsoft.Storage resources is forbidden", + resourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Storage/someStorageType/infraID", + expectedErr: api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "", "deletion of resource /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Storage/someStorageType/infraID is forbidden").Error(), + mocks: func(resources *mock_features.MockResourcesClient, loadBalancers *mock_network.MockLoadBalancersClient) { + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + env := mock_env.NewMockInterface(controller) + env.EXPECT().Location().AnyTimes().Return(location) + + networkLoadBalancers := mock_network.NewMockLoadBalancersClient(controller) + resources := mock_features.NewMockResourcesClient(controller) + tt.mocks(resources, networkLoadBalancers) + + a := azureActions{ + log: logrus.NewEntry(logrus.StandardLogger()), + env: env, + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/%s", subscription, clusterRG), + }, + }, + }, + loadBalancers: networkLoadBalancers, + resources: resources, + } + + ctx := context.Background() + + err := a.ResourceDeleteAndWait(ctx, tt.resourceID) + utilerror.AssertErrorMessage(t, err, tt.expectedErr) + }) + } +} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 1b3e1739332..f7e366eff53 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -327,6 +327,7 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/drainnode", f.postAdminOpenShiftClusterDrainNode) r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/etcdcertificaterenew", f.postAdminOpenShiftClusterEtcdCertificateRenew) + r.With(f.maintenanceMiddleware.UnplannedMaintenanceSignal).Post("/deletemanagedresource", f.postAdminOpenShiftDeleteManagedResource) }) }) diff --git a/pkg/util/azureclient/mgmt/features/resources_addons.go b/pkg/util/azureclient/mgmt/features/resources_addons.go index 52fb56d77c4..9b290bc7fc7 100644 --- a/pkg/util/azureclient/mgmt/features/resources_addons.go +++ b/pkg/util/azureclient/mgmt/features/resources_addons.go @@ -14,6 +14,7 @@ import ( type ResourcesClientAddons interface { Client() autorest.Client ListByResourceGroup(ctx context.Context, resourceGroupName string, filter string, expand string, top *int32) ([]mgmtfeatures.GenericResourceExpanded, error) + DeleteByIDAndWait(ctx context.Context, resourceID string, apiVersion string) error } func (c *resourcesClient) Client() autorest.Client { @@ -36,3 +37,12 @@ func (c *resourcesClient) ListByResourceGroup(ctx context.Context, resourceGroup return resources, nil } + +func (c *resourcesClient) DeleteByIDAndWait(ctx context.Context, resourceID string, apiVersion string) error { + future, err := c.DeleteByID(ctx, resourceID, apiVersion) + if err != nil { + return err + } + + return future.WaitForCompletionRef(ctx, c.Client()) +} diff --git a/pkg/util/loadbalancer/loadbalancer.go b/pkg/util/loadbalancer/loadbalancer.go new file mode 100644 index 00000000000..cefdd518cce --- /dev/null +++ b/pkg/util/loadbalancer/loadbalancer.go @@ -0,0 +1,30 @@ +package loadbalancer + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "fmt" + "strings" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" +) + +func RemoveFrontendIPConfiguration(lb *mgmtnetwork.LoadBalancer, resourceID string) error { + newFrontendIPConfig := make([]mgmtnetwork.FrontendIPConfiguration, 0, len(*lb.FrontendIPConfigurations)) + for _, fipConfig := range *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations { + if strings.EqualFold(*fipConfig.ID, resourceID) { + if isFrontendIPConfigReferenced(fipConfig) { + return fmt.Errorf("frontend IP Configuration %s has external references, remove the external references prior to removing the frontend IP configuration", resourceID) + } + continue + } + newFrontendIPConfig = append(newFrontendIPConfig, fipConfig) + } + lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &newFrontendIPConfig + return nil +} + +func isFrontendIPConfigReferenced(fipConfig mgmtnetwork.FrontendIPConfiguration) bool { + return fipConfig.LoadBalancingRules != nil || fipConfig.InboundNatPools != nil || fipConfig.InboundNatRules != nil || fipConfig.OutboundRules != nil +} diff --git a/pkg/util/loadbalancer/loadbalancer_test.go b/pkg/util/loadbalancer/loadbalancer_test.go new file mode 100644 index 00000000000..6b02d0106dd --- /dev/null +++ b/pkg/util/loadbalancer/loadbalancer_test.go @@ -0,0 +1,134 @@ +package loadbalancer + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "fmt" + "testing" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/go-autorest/autorest/to" + "github.com/stretchr/testify/assert" + + utilerror "github.com/Azure/ARO-RP/test/util/error" +) + +var infraID = "infraID" +var location = "eastus" +var publicIngressFIPConfigID = to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973") +var originalLB = mgmtnetwork.LoadBalancer{ + Sku: &mgmtnetwork.LoadBalancerSku{ + Name: mgmtnetwork.LoadBalancerSkuNameStandard, + }, + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + Name: to.StringPtr("public-lb-ip-v4"), + }, + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: publicIngressFIPConfigID, + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + { + Name: to.StringPtr("adce98f85c7dd47c5a21263a5e39c083"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/adce98f85c7dd47c5a21263a5e39c083"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-adce98f85c7dd47c5a21263a5e39c083"), + }, + }, + }, + }, + }, + Name: to.StringPtr(infraID), + Type: to.StringPtr("Microsoft.Network/loadBalancers"), + Location: to.StringPtr(location), +} + +func TestRemoveLoadBalancerFrontendIPConfiguration(t *testing.T) { + // Run tests + for _, tt := range []struct { + name string + fipResourceID string + currentLB mgmtnetwork.LoadBalancer + expectedLB mgmtnetwork.LoadBalancer + expectedErr string + }{ + { + name: "remove frontend ip config", + fipResourceID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/adce98f85c7dd47c5a21263a5e39c083", + currentLB: originalLB, + expectedLB: mgmtnetwork.LoadBalancer{ + Sku: &mgmtnetwork.LoadBalancerSku{ + Name: mgmtnetwork.LoadBalancerSkuNameStandard, + }, + LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{ + { + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-pip-v4"), + }, + }, + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/public-lb-ip-v4"), + Name: to.StringPtr("public-lb-ip-v4"), + }, + { + Name: to.StringPtr("ae3506385907e44eba9ef9bf76eac973"), + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/loadBalancers/infraID/frontendIPConfigurations/ae3506385907e44eba9ef9bf76eac973"), + FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{ + LoadBalancingRules: &[]mgmtnetwork.SubResource{ + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-80"), + }, + { + ID: to.StringPtr("ae3506385907e44eba9ef9bf76eac973-TCP-443"), + }, + }, + PublicIPAddress: &mgmtnetwork.PublicIPAddress{ + ID: to.StringPtr("/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clusterRG/providers/Microsoft.Network/publicIPAddresses/infraID-default-v4"), + }, + }, + }, + }, + }, + Name: to.StringPtr(infraID), + Type: to.StringPtr("Microsoft.Network/loadBalancers"), + Location: to.StringPtr(location), + }, + }, + { + name: "removal of frontend ip config fails when frontend ip config has references", + fipResourceID: *publicIngressFIPConfigID, + currentLB: originalLB, + expectedLB: originalLB, + expectedErr: fmt.Sprintf("frontend IP Configuration %s has external references, remove the external references prior to removing the frontend IP configuration", *publicIngressFIPConfigID), + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := RemoveFrontendIPConfiguration(&tt.currentLB, tt.fipResourceID) + assert.Equal(t, tt.expectedLB, tt.currentLB) + utilerror.AssertErrorMessage(t, err, tt.expectedErr) + }) + } +} diff --git a/pkg/util/mocks/adminactions/adminactions.go b/pkg/util/mocks/adminactions/adminactions.go index 855890c984f..e5ac3ab3a67 100644 --- a/pkg/util/mocks/adminactions/adminactions.go +++ b/pkg/util/mocks/adminactions/adminactions.go @@ -284,6 +284,20 @@ func (mr *MockAzureActionsMockRecorder) NICReconcileFailedState(arg0, arg1 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NICReconcileFailedState", reflect.TypeOf((*MockAzureActions)(nil).NICReconcileFailedState), arg0, arg1) } +// ResourceDeleteAndWait mocks base method. +func (m *MockAzureActions) ResourceDeleteAndWait(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceDeleteAndWait", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ResourceDeleteAndWait indicates an expected call of ResourceDeleteAndWait. +func (mr *MockAzureActionsMockRecorder) ResourceDeleteAndWait(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceDeleteAndWait", reflect.TypeOf((*MockAzureActions)(nil).ResourceDeleteAndWait), arg0, arg1) +} + // ResourceGroupHasVM mocks base method. func (m *MockAzureActions) ResourceGroupHasVM(arg0 context.Context, arg1 string) (bool, error) { m.ctrl.T.Helper() diff --git a/pkg/util/mocks/azureclient/mgmt/features/features.go b/pkg/util/mocks/azureclient/mgmt/features/features.go index 80e23fac081..b122c69ff64 100644 --- a/pkg/util/mocks/azureclient/mgmt/features/features.go +++ b/pkg/util/mocks/azureclient/mgmt/features/features.go @@ -294,6 +294,20 @@ func (mr *MockResourcesClientMockRecorder) DeleteByID(arg0, arg1, arg2 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteByID", reflect.TypeOf((*MockResourcesClient)(nil).DeleteByID), arg0, arg1, arg2) } +// DeleteByIDAndWait mocks base method. +func (m *MockResourcesClient) DeleteByIDAndWait(arg0 context.Context, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteByIDAndWait", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteByIDAndWait indicates an expected call of DeleteByIDAndWait. +func (mr *MockResourcesClientMockRecorder) DeleteByIDAndWait(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteByIDAndWait", reflect.TypeOf((*MockResourcesClient)(nil).DeleteByIDAndWait), arg0, arg1, arg2) +} + // GetByID mocks base method. func (m *MockResourcesClient) GetByID(arg0 context.Context, arg1, arg2 string) (features.GenericResource, error) { m.ctrl.T.Helper() diff --git a/test/e2e/adminapi_delete_managedresource.go b/test/e2e/adminapi_delete_managedresource.go new file mode 100644 index 00000000000..094f2f6d5a8 --- /dev/null +++ b/test/e2e/adminapi_delete_managedresource.go @@ -0,0 +1,128 @@ +package e2e + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/util/stringutils" +) + +var loadBalancerService = corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{ + { + Name: "service-443", + Protocol: corev1.ProtocolTCP, + Port: int32(443), + }, + }, + }, +} + +var _ = Describe("[Admin API] Delete managed resource action", func() { + BeforeEach(skipIfNotInDevelopmentEnv) + + It("should be possible to delete managed cluster resources", func(ctx context.Context) { + var service *corev1.Service + var lbRuleID string + var fipConfigID string + var pipAddressID string + + By("creating a test service of type loadbalancer") + _, err := clients.Kubernetes.CoreV1().Services("default").Create(ctx, &loadBalancerService, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + defer func() { + By("cleaning up the k8s loadbalancer service") + err := clients.Kubernetes.CoreV1().Services("default").Delete(ctx, "test", metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + // wait for deletion to prevent flakes on retries + Eventually(func(g Gomega, ctx context.Context) { + _, err = clients.Kubernetes.CoreV1().Services("default").Get(ctx, "test", metav1.GetOptions{}) + g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "expect Service to be deleted") + }).WithContext(ctx).WithTimeout(DefaultEventuallyTimeout).Should(Succeed()) + }() + + // wait for ingress IP to be assigned as this indicate the service is ready + Eventually(func(g Gomega, ctx context.Context) { + service, err = clients.Kubernetes.CoreV1().Services("default").Get(ctx, "test", metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(service.Status.LoadBalancer.Ingress).To(HaveLen(1)) + }).WithContext(ctx).WithTimeout(DefaultEventuallyTimeout).Should(Succeed()) + + By("getting the newly created k8s service frontend IP configuration") + oc, err := clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName) + Expect(err).NotTo(HaveOccurred()) + + rgName := stringutils.LastTokenByte(*oc.OpenShiftClusterProperties.ClusterProfile.ResourceGroupID, '/') + lbName, err := getInfraID(ctx) + Expect(err).NotTo(HaveOccurred()) + + lb, err := clients.LoadBalancers.Get(ctx, rgName, lbName, "") + Expect(err).NotTo(HaveOccurred()) + + for _, fipConfig := range *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations { + if !strings.Contains(*fipConfig.PublicIPAddress.ID, "default-v4") && !strings.Contains(*fipConfig.PublicIPAddress.ID, "pip-v4") { + lbRuleID = *(*fipConfig.LoadBalancingRules)[0].ID + fipConfigID = *fipConfig.ID + pipAddressID = *fipConfig.PublicIPAddress.ID + } + } + + By("deleting the associated loadbalancer rule") + testDeleteManagedResourceOK(ctx, lbRuleID) + + By("deleting the associated frontend ip config") + testDeleteManagedResourceOK(ctx, fipConfigID) + + By("deleting the associated public ip address") + testDeleteManagedResourceOK(ctx, pipAddressID) + }) + + It("should NOT be possible to delete a resource not within the cluster's managed resource group", func(ctx context.Context) { + By("trying to delete the master subnet") + oc, err := clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName) + Expect(err).NotTo(HaveOccurred()) + + resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/deletemanagedresource", url.Values{"managedResourceID": []string{*oc.OpenShiftClusterProperties.MasterProfile.SubnetID}}, true, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("should NOT be possible to delete the private link service in the cluster's managed resource group", func(ctx context.Context) { + By("trying to delete the private link service") + oc, err := clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName) + Expect(err).NotTo(HaveOccurred()) + + // Fake name prevents accidently deleting the PLS but still validates gaurdrail logic works. + plsResourceID := fmt.Sprintf("%s/providers/Microsoft.Network/PrivateLinkServices/%s", *oc.OpenShiftClusterProperties.ClusterProfile.ResourceGroupID, "fake-pls") + + resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/deletemanagedresource", url.Values{"managedResourceID": []string{plsResourceID}}, true, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) +}) + +func testDeleteManagedResourceOK(ctx context.Context, resourceID string) { + resp, err := adminRequest(ctx, http.MethodPost, "/admin"+clusterResourceID+"/deletemanagedresource", url.Values{"managedResourceID": []string{resourceID}}, true, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) +} diff --git a/test/e2e/update.go b/test/e2e/update.go index 068a7f51ce7..123715deab3 100644 --- a/test/e2e/update.go +++ b/test/e2e/update.go @@ -86,7 +86,7 @@ var _ = Describe("Update cluster Managed Outbound IPs", func() { oc, err := clients.OpenshiftClustersPreview.Get(ctx, vnetResourceGroup, clusterName) Expect(err).NotTo(HaveOccurred()) - lbName, err = getPublicLoadBalancerName(ctx) + lbName, err = getInfraID(ctx) Expect(err).NotTo(HaveOccurred()) rgName = stringutils.LastTokenByte(*oc.ClusterProfile.ResourceGroupID, '/') @@ -136,7 +136,7 @@ var _ = Describe("Update cluster Managed Outbound IPs", func() { }) }) -func getPublicLoadBalancerName(ctx context.Context) (string, error) { +func getInfraID(ctx context.Context) (string, error) { co, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{}) if err != nil { return "", err