Delete the vs/lb pool created for pre-created VPC

pkg/clean/clean.go

@@ -174,8 +174,8 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Clien
 		AddCleanupService(wrapInitializeSubnetService(commonService)).
 		AddCleanupService(wrapInitializeSecurityPolicy(commonService)).
 		AddCleanupService(wrapInitializeStaticRoute(commonService)).
-		AddCleanupService(wrapInitializeIPAddressAllocation(commonService)).
-		AddCleanupService(wrapInitializeVPC(commonService))
+		AddCleanupService(wrapInitializeVPC(commonService)).
+		AddCleanupService(wrapInitializeIPAddressAllocation(commonService))
 
 	return cleanupService, nil
 }

pkg/nsx/client.go

@@ -90,6 +90,8 @@ type Client struct {
 	RealizedStateClient            realized_state.RealizedEntitiesClient
 	IPAddressAllocationClient      vpcs.IpAddressAllocationsClient
 	VPCLBSClient                   vpcs.VpcLbsClient
+	VpcLbVirtualServersClient      vpcs.VpcLbVirtualServersClient
+	VpcLbPoolsClient               vpcs.VpcLbPoolsClient
 	ProjectClient                  orgs.ProjectsClient
 	TransitGatewayClient           projects.TransitGatewaysClient
 	TransitGatewayAttachmentClient transit_gateways.AttachmentsClient
@@ -182,6 +184,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
 	ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster))
 	vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))
+	vpcLbVirtualServersClient := vpcs.NewVpcLbVirtualServersClient(restConnector(cluster))
+	vpcLbPoolsClient := vpcs.NewVpcLbPoolsClient(restConnector(cluster))
 
 	vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
 	vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
@@ -228,6 +232,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 		VPCSecurityClient:              vpcSecurityClient,
 		VPCRuleClient:                  vpcRuleClient,
 		VPCLBSClient:                   vpcLBSClient,
+		VpcLbVirtualServersClient:      vpcLbVirtualServersClient,
+		VpcLbPoolsClient:               vpcLbPoolsClient,
 		ProjectClient:                  projectClient,
 		NSXChecker:                     *nsxChecker,
 		NSXVerChecker:                  *nsxVersionChecker,

pkg/nsx/services/common/types.go

@@ -21,10 +21,12 @@ const (
 	MaxIdLength                        int    = 255
 	MaxNameLength                      int    = 255
 	MaxSubnetNameLength                int    = 80
+	VPCLbResourcePathMinSegments       int    = 8
 	PriorityNetworkPolicyAllowRule     int    = 2010
 	PriorityNetworkPolicyIsolationRule int    = 2090
 	TagScopeNCPCluster                 string = "ncp/cluster"
 	TagScopeNCPProjectUID              string = "ncp/project_uid"
+	TagScopeNCPCreateFor               string = "ncp/created_for"
 	TagScopeNCPVIFProjectUID           string = "ncp/vif_project_uid"
 	TagScopeNCPPod                     string = "ncp/pod"
 	TagScopeNCPVNETInterface           string = "ncp/vnet_interface"
@@ -175,6 +177,8 @@ var (
 	ResourceTypeLBSourceIpPersistenceProfile = "LBSourceIpPersistenceProfile"
 	ResourceTypeLBHttpMonitorProfile         = "LBHttpMonitorProfile"
 	ResourceTypeLBTcpMonitorProfile          = "LBTcpMonitorProfile"
+	ResourceTypeLBVirtualServer              = "LBVirtualServer"
+	ResourceTypeLBPool                       = "LBPool"
 
 	// ResourceTypeClusterControlPlane is used by NSXServiceAccountController
 	ResourceTypeClusterControlPlane = "clustercontrolplane"

pkg/nsx/services/vpc/builder.go

@@ -32,6 +32,20 @@ func generateLBSKey(lbs model.LBService) (string, error) {
 	return combineVPCIDAndLBSID(vpcID, *lbs.Id), nil
 }
 
+func generateVirtualServerKey(vs model.LBVirtualServer) (string, error) {
+	if vs.Path == nil || *vs.Path == "" {
+		return "", fmt.Errorf("LBVirtualServer path is nil or empty")
+	}
+	return *vs.Path, nil
+}
+
+func generatePoolKey(pool model.LBPool) (string, error) {
+	if pool.Path == nil || *pool.Path == "" {
+		return "", fmt.Errorf("LBPool path is nil or empty")
+	}
+	return *pool.Path, nil
+}
+
 func combineVPCIDAndLBSID(vpcID, lbsID string) string {
 	return fmt.Sprintf("%s_%s", vpcID, lbsID)
 }

pkg/nsx/services/vpc/store.go

@@ -15,6 +15,10 @@ func keyFunc(obj interface{}) (string, error) {
 		return *v.Id, nil
 	case *model.LBService:
 		return generateLBSKey(*v)
+	case *model.LBVirtualServer:
+		return generateVirtualServerKey(*v)
+	case *model.LBPool:
+		return generatePoolKey(*v)
 	default:
 		return "", errors.New("keyFunc doesn't support unknown type")
 	}

pkg/nsx/services/vpc/vpc.go

@@ -244,6 +244,13 @@ func (s *VPCService) addClusterTag(query string) string {
 	return query + " AND " + tagParam
 }
 
+func (s *VPCService) addNCPCreatedForTag(query string) string {
+	tagScopeClusterKey := strings.Replace(common.TagScopeNCPCreateFor, "/", "\\/", -1)
+	tagScopeClusterValue := strings.Replace(common.TagValueSLB, ":", "\\:", -1)
+	tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue)
+	return query + " AND " + tagParam
+}
+
 func (s *VPCService) ListCert() []model.TlsCertificate {
 	store := &ResourceStore{ResourceStore: common.ResourceStore{
 		Indexer:     cache.NewIndexer(keyFunc, cache.Indexers{}),
@@ -434,6 +441,84 @@ func (s *VPCService) DeleteLBMonitorProfile(id string) error {
 	log.Info("successfully deleted NCP created lbMonitorProfile", "lbMonitorProfile", id)
 	return nil
 }
+func (s *VPCService) ListLBVirtualServer() []model.LBVirtualServer {
+	store := &ResourceStore{ResourceStore: common.ResourceStore{
+		Indexer:     cache.NewIndexer(keyFunc, cache.Indexers{}),
+		BindingType: model.LBVirtualServerBindingType(),
+	}}
+	query := fmt.Sprintf("(%s:%s)",
+		common.ResourceType, common.ResourceTypeLBVirtualServer)
+	query = s.addClusterTag(query)
+	query = s.addNCPCreatedForTag(query)
+	count, searcherr := s.SearchResource("", query, store, nil)
+	if searcherr != nil {
+		log.Error(searcherr, "failed to query LBVirtualServer", "query", query)
+	} else {
+		log.V(1).Info("query LBVirtualServer", "count", count)
+	}
+	lbVirtualServers := store.List()
+	lbVirtualServersSet := []model.LBVirtualServer{}
+	for _, lbVirtualServer := range lbVirtualServers {
+		lbVirtualServersSet = append(lbVirtualServersSet, *lbVirtualServer.(*model.LBVirtualServer))
+	}
+	return lbVirtualServersSet
+}
+
+func (s *VPCService) DeleteLBVirtualServer(path string) error {
+	lbVirtualServersClient := s.NSXClient.VpcLbVirtualServersClient
+	boolValue := false
+	paths := strings.Split(path, "/")
+
+	if len(paths) < common.VPCLbResourcePathMinSegments {
+		// skip virtual server under infra
+		log.Info("failed to parse virtual server path", "path", path)
+		return nil
+	}
+	if err := lbVirtualServersClient.Delete(paths[2], paths[4], paths[6], paths[8], &boolValue); err != nil {
+		return err
+	}
+	log.Info("successfully deleted NCP created lbVirtualServer", "lbVirtualServer", path)
+	return nil
+}
+
+func (s *VPCService) ListLBPool() []model.LBPool {
+	store := &ResourceStore{ResourceStore: common.ResourceStore{
+		Indexer:     cache.NewIndexer(keyFunc, cache.Indexers{}),
+		BindingType: model.LBPoolBindingType(),
+	}}
+	query := fmt.Sprintf("(%s:%s)",
+		common.ResourceType, common.ResourceTypeLBPool)
+	query = s.addClusterTag(query)
+	query = s.addNCPCreatedForTag(query)
+	count, searcherr := s.SearchResource("", query, store, nil)
+	if searcherr != nil {
+		log.Error(searcherr, "failed to query LBPool", "query", query)
+	} else {
+		log.V(1).Info("query LBPool", "count", count)
+	}
+	lbPools := store.List()
+	lbPoolsSet := []model.LBPool{}
+	for _, lbPool := range lbPools {
+		lbPoolsSet = append(lbPoolsSet, *lbPool.(*model.LBPool))
+	}
+	return lbPoolsSet
+}
+
+func (s *VPCService) DeleteLBPool(path string) error {
+	lbPoolsClient := s.NSXClient.VpcLbPoolsClient
+	boolValue := false
+	paths := strings.Split(path, "/")
+	if len(paths) < 8 {
+		// skip lb pool under infra
+		log.Info("failed to parse lb pool path", "path", path)
+		return nil
+	}
+	if err := lbPoolsClient.Delete(paths[2], paths[4], paths[6], paths[8], &boolValue); err != nil {
+		return err
+	}
+	log.Info("successfully deleted NCP created lbPool", "lbPool", path)
+	return nil
+}
 
 func (s *VPCService) IsSharedVPCNamespaceByNS(ns string) (bool, error) {
 	shared_ns, err := s.getSharedVPCNamespaceFromNS(ns)
@@ -921,6 +1006,33 @@ func (s *VPCService) Cleanup(ctx context.Context) error {
 			}
 		}
 	}
+
+	// Clean vs/lb pool created for pre-created vpc
+	lbVirtualServers := s.ListLBVirtualServer()
+	log.Info("cleaning up lbVirtualServers", "Count", len(lbVirtualServers))
+	for _, lbVirtualServer := range lbVirtualServers {
+		select {
+		case <-ctx.Done():
+			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
+		default:
+			if err := s.DeleteLBVirtualServer(*lbVirtualServer.Path); err != nil {
+				return err
+			}
+		}
+	}
+
+	lbPools := s.ListLBPool()
+	log.Info("cleaning up lbPools", "Count", len(lbPools))
+	for _, lbPool := range lbPools {
+		select {
+		case <-ctx.Done():
+			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
+		default:
+			if err := s.DeleteLBPool(*lbPool.Path); err != nil {
+				return err
+			}
+		}
+	}
 	// We don't clean client_ssl_profile as client_ssl_profile is not created by ncp or nsx-operator
 	return nil
 }