diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index 1aa0e7e91..865c988aa 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -69,27 +69,6 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) return common.ResultRequeueAfter10sec, err } - isShared, err := r.Service.IsSharedVPCNamespaceByNS(obj.GetNamespace()) - if err != nil { - log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace()) - return common.ResultRequeue, err - } - if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && !isShared { - err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace) - if err != nil { - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - VPCPath: *createdVpc.Path, - DefaultSNATIP: "", - LoadBalancerIPAddresses: "", - PrivateIPs: nc.PrivateIPs, - } - log.Error(err, "update avi rule failed, would retry exponentially", "NetworkInfo", req.NamespacedName, "state", state) - // updateFail(r, &ctx, obj, &err, r.Client, state) - // return common.ResultRequeueAfter10sec, err - } - } - snatIP, path, cidr := "", "", "" parts := strings.Split(nc.VPCConnectivityProfile, "/") if len(parts) < 1 { diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index a4968e3d6..097a73c13 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -4,16 +4,13 @@ import ( "context" "errors" "fmt" - "math" "net" "strings" "sync" - "github.com/vmware/vsphere-automation-sdk-go/runtime/data" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -710,245 +707,6 @@ func (s *VPCService) Cleanup(ctx context.Context) error { return nil } -func (service *VPCService) needUpdateRule(rule *model.Rule, externalCIDRs []string) bool { - des := rule.DestinationGroups - currentDesSet := sets.Set[string]{} - for _, group := range des { - currentDesSet.Insert(group) - } - if len(externalCIDRs) != len(currentDesSet) { - return true - } - for _, cidr := range externalCIDRs { - if !currentDesSet.Has(cidr) { - return true - } - } - return false -} - -func (service *VPCService) getIpblockCidr(blocks []string) (result []string, err error) { - for _, cidr := range blocks { - ipblock := service.PubIpblockStore.GetByKey(cidr) - if ipblock == nil { - // in case VPC using the new ipblock, search the ipblock from nsxt - // return error, and retry next time when the ipblock is synced into store - err = errors.New("ipblock not found") - log.Error(err, "failed to get public ipblock", "path", cidr) - query := fmt.Sprintf("%s:%s AND visibility:EXTERNAL", common.ResourceType, common.ResourceTypeIPBlock) - count, searcherr := service.SearchResource(common.ResourceTypeIPBlock, query, service.PubIpblockStore, nil) - if searcherr != nil { - log.Error(searcherr, "failed to query public ipblock", "query", query) - } else { - log.V(1).Info("query public ipblock", "count", count) - } - return - } else { - result = append(result, *ipblock.Cidr) - } - } - return -} - -func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace string) error { - if !enableAviAllowRule { - return nil - } - if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { - log.Info("avi rule cannot be created or updated due to no DFW license") - return nil - } - vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path) - if err != nil { - log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path) - return err - } - orgId := vpcInfo.OrgID - projectId := vpcInfo.ProjectID - ruleId := AviSEIngressAllowRuleId - groupId := VPCAviSEGroupId - spId := VpcDefaultSecurityPolicyId - - if !service.checkAVISecurityPolicyExist(orgId, projectId, *vpc.Id, spId) { - return errors.New("avi security policy not found") - } - allowrule, err := service.getAVIAllowRule(orgId, projectId, *vpc.Id, spId, ruleId) - if err != nil { - log.Info("avi rule is not found, creating") - } - externalCIDRs, err := service.getIpblockCidr(vpc.ExternalIpv4Blocks) - if err != nil { - return err - } - log.Info("avi rule get external cidr", "cidr", externalCIDRs) - if allowrule != nil { - if !service.needUpdateRule(allowrule, externalCIDRs) { - log.Info("avi rule is not changed, skip updating avi rule") - return nil - } else { - log.Info("avi rule changed", "previous", allowrule.DestinationGroups, "current", externalCIDRs) - } - } - - group, err := service.getorCreateAVIGroup(orgId, projectId, *vpc.Id, groupId) - if err != nil { - log.Error(err, "failed to get avi group", "group", groupId) - return err - } - - newrule, err := service.buildAVIAllowRule(vpc, externalCIDRs, *group.Path, ruleId, projectId) - log.Info("creating avi rule", "rule", newrule) - if err != nil { - log.Error(err, "failed to build avi rule", "rule", newrule) - return err - } - - err = service.NSXClient.VPCRuleClient.Patch(orgId, projectId, *vpc.Id, spId, *newrule.Id, *newrule) - err = nsxutil.NSXApiError(err) - if err != nil { - log.Error(err, "failed to create avi rule", "rule", newrule) - return err - } - nsxrule, err := service.NSXClient.VPCRuleClient.Get(orgId, projectId, *vpc.Id, spId, *newrule.Id) - err = nsxutil.NSXApiError(err) - if err != nil { - log.Error(err, "failed to get avi rule", "rule", nsxrule) - return err - } - service.RuleStore.Add(&nsxrule) - log.Info("created avi rule successfully") - return nil -} - -func (service *VPCService) getorCreateAVIGroup(orgId string, projectId string, vpcId string, groupId string) (*model.Group, error) { - groupPtr, err := service.getAVIGroup(orgId, projectId, vpcId, groupId) - if err != nil { - log.Info("create avi group", "group", groupId) - groupPtr, err = service.createAVIGroup(orgId, projectId, vpcId, groupId) - if err != nil { - log.Error(err, "failed to create avi group", "group", groupId) - return groupPtr, err - } - service.GroupStore.Add(groupPtr) - } - return groupPtr, err -} - -func (service *VPCService) buildAVIGroupTag(vpcId string) []model.Tag { - return []model.Tag{ - { - Scope: common.String(common.TagScopeCluster), - Tag: common.String(service.NSXConfig.Cluster), - }, - { - Scope: common.String(common.TagScopeVersion), - Tag: common.String(strings.Join(common.TagValueVersion, ".")), - }, - { - Scope: common.String(common.TagScopeGroupType), - Tag: common.String(common.TagValueGroupAvi), - }, - } -} - -func (service *VPCService) createAVIGroup(orgId string, projectId string, vpcId string, groupId string) (*model.Group, error) { - group := model.Group{} - group.Tags = service.buildAVIGroupTag(vpcId) - expression := service.buildExpression("Condition", "VpcSubnet", "AVI_SUBNET_LB|", "Tag", "EQUALS", "EQUALS") - group.Expression = []*data.StructValue{expression} - group.DisplayName = common.String(groupId) - - err := service.NSXClient.VpcGroupClient.Patch(orgId, projectId, vpcId, groupId, group) - err = nsxutil.NSXApiError(err) - if err != nil { - return &group, err - } - nsxgroup, err := service.NSXClient.VpcGroupClient.Get(orgId, projectId, vpcId, groupId) - err = nsxutil.NSXApiError(err) - return &nsxgroup, err -} - -func (service *VPCService) buildExpression(resource_type, member_type, value, key, operator, scope_op string) *data.StructValue { - return data.NewStructValue( - "", - map[string]data.DataValue{ - "resource_type": data.NewStringValue(resource_type), - "member_type": data.NewStringValue(member_type), - "value": data.NewStringValue(value), - "key": data.NewStringValue(key), - "operator": data.NewStringValue(operator), - "scope_operator": data.NewStringValue(scope_op), - }, - ) -} - -func (service *VPCService) buildAVIAllowRule(obj *model.Vpc, externalCIDRs []string, groupId, ruleId, projectId string) (*model.Rule, error) { - rule := &model.Rule{} - rule.Action = common.String(model.Rule_ACTION_ALLOW) - rule.Direction = common.String(model.Rule_DIRECTION_IN_OUT) - rule.Scope = append(rule.Scope, groupId) - rule.SequenceNumber = common.Int64(math.MaxInt32 - 1) - rule.DestinationGroups = externalCIDRs - rule.SourceGroups = append(rule.SourceGroups, "Any") - name := fmt.Sprintf("PROJECT-%s-VPC-%s-%s", projectId, *obj.Id, ruleId) - rule.DisplayName = common.String(name) - rule.Id = common.String(ruleId) - rule.Services = []string{"ANY"} - rule.IsDefault = common.Bool(true) - tags := []model.Tag{ - { - Scope: common.String(common.TagScopeCluster), - Tag: common.String(service.NSXConfig.Cluster), - }, - { - Scope: common.String(common.TagScopeVersion), - Tag: common.String(strings.Join(common.TagValueVersion, ".")), - }, - } - rule.Tags = tags - return rule, nil -} - -func (service *VPCService) getAVIAllowRule(orgId string, projectId string, vpcId string, spId string, ruleId string) (*model.Rule, error) { - key := fmt.Sprintf(RuleKey, orgId, projectId, vpcId, spId, ruleId) - rule := service.RuleStore.GetByKey(key) - if rule == nil { - log.Info("avi rule not found", "key", key) - return nil, errors.New("avi rule not found") - } - return rule, nil -} - -func (service *VPCService) getAVIGroup(orgId string, projectId string, vpcId string, groupId string) (*model.Group, error) { - key := fmt.Sprintf(GroupKey, orgId, projectId, vpcId, groupId) - group := service.GroupStore.GetByKey(key) - var err error - if group == nil { - log.Info("avi se group not found", "key", key) - err = errors.New("avi se group not found") - } - return group, err -} - -// checkAVISecurityPolicyExist returns true if security policy for that VPC already exists -// this security policy created by NSXT once VPC created -// if not found, wait until it created -func (service *VPCService) checkAVISecurityPolicyExist(orgId string, projectId string, vpcId string, spId string) bool { - key := fmt.Sprintf(SecurityPolicyKey, orgId, projectId, vpcId, spId) - sp := service.SecurityPolicyStore.GetByKey(key) - if sp != nil { - return true - } - nsxtsp, err := service.NSXClient.VPCSecurityClient.Get(orgId, projectId, vpcId, spId) - err = nsxutil.NSXApiError(err) - if err != nil { - log.Error(err, "failed to get avi security policy", "key", key) - return false - } - service.SecurityPolicyStore.Add(&nsxtsp) - return true -} - func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { var VPCInfoList []common.VPCResourceInfo vpcs := service.GetVPCsByNamespace(ns) // Transparently call the VPCService.GetVPCsByNamespace method diff --git a/pkg/nsx/services/vpc/vpc_test.go b/pkg/nsx/services/vpc/vpc_test.go index 704242906..1a53df8b2 100644 --- a/pkg/nsx/services/vpc/vpc_test.go +++ b/pkg/nsx/services/vpc/vpc_test.go @@ -3,7 +3,6 @@ package vpc import ( "context" "errors" - "fmt" "reflect" "testing" @@ -21,7 +20,6 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -375,145 +373,3 @@ func (ruleClient *MockRuleClient) Update(orgIdParam string, projectIdParam strin func (ruleClient *MockRuleClient) Revise(orgIdParam string, projectIdParam string, vpcIdParam string, securityPolicyIdParam string, ruleIdParam string, ruleParam model.Rule, anchorPathParam *string, operationParam *string) (model.Rule, error) { return model.Rule{}, ruleClient.Err } - -func TestCreateOrUpdateAVIRule(t *testing.T) { - aviRuleCacheIndexer := cache.NewIndexer(keyFuncAVI, nil) - resourceStore := common.ResourceStore{ - Indexer: aviRuleCacheIndexer, - BindingType: model.VpcBindingType(), - } - ruleStore := &AviRuleStore{ResourceStore: resourceStore} - resourceStore1 := common.ResourceStore{ - Indexer: aviRuleCacheIndexer, - BindingType: model.GroupBindingType(), - } - groupStore := &AviGroupStore{ResourceStore: resourceStore1} - resourceStore2 := common.ResourceStore{ - Indexer: aviRuleCacheIndexer, - BindingType: model.SecurityPolicyBindingType(), - } - spStore := &AviSecurityPolicyStore{ResourceStore: resourceStore2} - - service := &VPCService{ - Service: common.Service{NSXClient: nil}, - VPCNetworkConfigStore: VPCNetworkInfoStore{ - VPCNetworkConfigMap: map[string]common.VPCNetworkConfigInfo{}, - }, - VPCNSNetworkConfigStore: VPCNsNetworkConfigStore{ - VPCNSNetworkConfigMap: map[string]string{}, - }, - } - - service.RuleStore = ruleStore - service.GroupStore = groupStore - service.SecurityPolicyStore = spStore - - ns1 := "test-ns-1" - tag1 := []model.Tag{ - { - Scope: &tagScopeCluster, - Tag: &cluster, - }, - { - Scope: &tagScopeNamespace, - Tag: &ns1, - }, - } - path1 := "/orgs/default/projects/project_1/vpcs/vpc1" - vpc1 := model.Vpc{ - Path: &path1, - DisplayName: &vpcName1, - Id: &vpcID1, - Tags: tag1, - IpAddressType: &IPv4Type, - PrivateIpv4Blocks: []string{"1.1.1.0/24"}, - ExternalIpv4Blocks: []string{"2.2.2.0/24"}, - } - - // feature not supported - enableAviAllowRule = false - err := service.CreateOrUpdateAVIRule(&vpc1, ns1) - assert.Equal(t, err, nil) - - // enable feature - enableAviAllowRule = true - spClient := MockSecurityPoliciesClient{} - - service.NSXClient = &nsx.Client{} - service.NSXClient.VPCSecurityClient = &spClient - service.NSXConfig = &config.NSXOperatorConfig{} - service.NSXConfig.CoeConfig = &config.CoeConfig{} - service.NSXConfig.Cluster = "k8scl_one" - sppath1 := "/orgs/default/projects/project_1/vpcs/vpc1/security-policies/sp1" - sp := model.SecurityPolicy{ - Path: &sppath1, - } - util.UpdateLicense(util.FeatureDFW, true) - - // security policy not found - spClient.SP = sp - notFound := errors.New("avi security policy not found") - spClient.Err = notFound - err = service.CreateOrUpdateAVIRule(&vpc1, ns1) - assert.Equal(t, err, notFound) - - // security policy found, get rule, failed to get external CIDR - rulepath1 := fmt.Sprintf("/orgs/default/projects/project_1/vpcs/ns-vpc-uid-1/security-policies/default-layer3-section/rules/%s", AviSEIngressAllowRuleId) - rule := model.Rule{ - Path: &rulepath1, - DestinationGroups: []string{"2.2.2.0/24"}, - } - ruleStore.Add(&rule) - spClient.Err = nil - resulterr := errors.New("get external ipblock failed") - patch := gomonkey.ApplyPrivateMethod(reflect.TypeOf(service), "getIpblockCidr", func(_ *VPCService, cidr []string) ([]string, error) { - return []string{}, resulterr - }) - err = service.CreateOrUpdateAVIRule(&vpc1, ns1) - patch.Reset() - assert.Equal(t, err, resulterr) - - // security policy found, get rule, get external CIDR which matched - spClient.Err = nil - resulterr = errors.New("get external ipblock failed") - patch = gomonkey.ApplyPrivateMethod(reflect.TypeOf(service), "getIpblockCidr", func(_ *VPCService, cidr []string) ([]string, error) { - return []string{"2.2.2.0/24"}, nil - }) - err = service.CreateOrUpdateAVIRule(&vpc1, ns1) - patch.Reset() - assert.Equal(t, err, nil) - - // security policy found, get external CIDR, create group failed - patch = gomonkey.ApplyPrivateMethod(reflect.TypeOf(service), "getIpblockCidr", func(_ *VPCService, cidr []string) ([]string, error) { - return []string{"192.168.0.0/16"}, nil - }) - defer patch.Reset() - groupClient := MockGroupClient{Err: nil} - service.NSXClient.VpcGroupClient = &groupClient - grouppath1 := "/orgs/default/projects/project_1/vpcs/vpc1/groups/group1" - group := model.Group{ - Path: &grouppath1, - } - groupClient.Group = group - groupClient.Err = errors.New("create avi group error") - service.NSXConfig = &config.NSXOperatorConfig{} - service.NSXConfig.CoeConfig = &config.CoeConfig{} - service.NSXConfig.Cluster = "k8scl-one" - err = service.CreateOrUpdateAVIRule(&vpc1, ns1) - assert.Equal(t, err, groupClient.Err) - - // security policy found, get external CIDR, create group, create rule failed - groupClient.Err = nil - ruleClient := MockRuleClient{} - service.NSXClient.VPCRuleClient = &ruleClient - - ruleClient.Rule = rule - ruleClient.Err = errors.New("create avi rule error") - err = service.CreateOrUpdateAVIRule(&vpc1, ns1) - assert.Equal(t, err, ruleClient.Err) - - // security policy found, get external CIDR, create group, create rule - ruleClient.Err = nil - err = service.CreateOrUpdateAVIRule(&vpc1, ns1) - assert.Equal(t, err, nil) -}