Switch to Polict API for Segment and Segment Ports

As NSX 9.0 intends to remove MP APIs, we need to switch to Policy API.
This patch makes the changes necessary for node controller Logical Switch
related APIs

The patch removes some stale code which is no longer used

Testing Done
------------
- Verified that the Segment Ports for nodes are updated by the operator
- Verified that the operator is running fine

Jira: NCP-601

go.mod

@@ -14,8 +14,8 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
 	github.com/vmware/go-vmware-nsxt v0.0.0-20201207175959-23201aae9cc3
-	github.com/vmware/vsphere-automation-sdk-go/runtime v0.2.0
-	github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.3.0
+	github.com/vmware/vsphere-automation-sdk-go/runtime v0.7.0
+	github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.12.0
 	gopkg.in/ini.v1 v1.51.0
 	k8s.io/api v0.18.2
 	k8s.io/apimachinery v0.18.3
@@ -45,11 +45,12 @@ require (
 	github.com/go-logr/logr v0.1.0 // indirect
 	github.com/go-logr/zapr v0.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
 	github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect
 	github.com/golang/protobuf v1.5.0 // indirect
 	github.com/google/go-cmp v0.5.5 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/google/uuid v1.1.1 // indirect
+	github.com/google/uuid v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.3.1 // indirect
 	github.com/gophercloud/gophercloud v0.6.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
@@ -66,7 +67,7 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/vmware/vsphere-automation-sdk-go/lib v0.2.0 // indirect
+	github.com/vmware/vsphere-automation-sdk-go/lib v0.7.0 // indirect
 	go.uber.org/atomic v1.6.0 // indirect
 	go.uber.org/multierr v1.5.0 // indirect
 	go.uber.org/zap v1.14.1 // indirect

go.sum

@@ -344,6 +344,8 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6
 github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog=
+github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -405,6 +407,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -836,10 +840,16 @@ github.com/vmware/go-vmware-nsxt v0.0.0-20201207175959-23201aae9cc3 h1:lPZIdo123
 github.com/vmware/go-vmware-nsxt v0.0.0-20201207175959-23201aae9cc3/go.mod h1:VEqcmf4Sp7gPB7z05QGyKVmn6xWppr7Nz8cVNvyC80o=
 github.com/vmware/vsphere-automation-sdk-go/lib v0.2.0 h1:noiASkafwxvCmmCtZK0f1pt06R8c+Fkt9TcZW7xpYy0=
 github.com/vmware/vsphere-automation-sdk-go/lib v0.2.0/go.mod h1://FsAiCrr+T/Eq2Uxtq8UPVPbZWV7iLIvvXK17rsIxE=
+github.com/vmware/vsphere-automation-sdk-go/lib v0.7.0 h1:pT+oqJ8FD5eUBQkl+e7LZwwtbwPvW5kDyyGXvt66gOM=
+github.com/vmware/vsphere-automation-sdk-go/lib v0.7.0/go.mod h1:f3+6YVZpNcK2pYyiQ94BoHWmjMj9BnYav0vNFuTiDVM=
 github.com/vmware/vsphere-automation-sdk-go/runtime v0.2.0 h1:AM5AK9cyiJWFIfxrh1U/kuRFh+A2pymCEGiXqAkPzzw=
 github.com/vmware/vsphere-automation-sdk-go/runtime v0.2.0/go.mod h1:M6pTKDrJrPlVG++lboLRf0bDYc3TJ2fsR+KOoWXfCns=
+github.com/vmware/vsphere-automation-sdk-go/runtime v0.7.0 h1:pSBxa9Agh6bgW8Hr0A1eQxuwnxGTnuAVox8iQb023hg=
+github.com/vmware/vsphere-automation-sdk-go/runtime v0.7.0/go.mod h1:qdzEFm2iK3dvlmm99EYYNxs70HbzuiHyENFD24Ps8fQ=
 github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.3.0 h1:Ekf0/umhKdr4N0oURDFlkhZHVm6w0eXzbsn6yc/vL+4=
 github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.3.0/go.mod h1:k9tf91B5Ah7gkaM2s+Z6nATmn6gKmgt8AqJ8RUiKLfo=
+github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.12.0 h1:+kcDO69bfIB87KZUAYQ4AqrXlnZhpZz+QwzIB+TseqU=
+github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.12.0/go.mod h1:upLH9b9zpG86P0wwO4+gREf0lBXr8gYcs7P1FRZ9n30=
 github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=

pkg/controller/node/node_controller.go

@@ -17,17 +17,16 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	nsxt "github.com/vmware/go-vmware-nsxt"
-	"github.com/vmware/go-vmware-nsxt/common"
-	nsxtmgr "github.com/vmware/go-vmware-nsxt/manager"
 
 	"github.com/vmware/nsx-container-plugin-operator/pkg/controller/sharedinfo"
 	"github.com/vmware/nsx-container-plugin-operator/pkg/controller/statusmanager"
 	operatortypes "github.com/vmware/nsx-container-plugin-operator/pkg/types"
 	"github.com/vmware/vsphere-automation-sdk-go/runtime/core"
-	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
 	vspherelog "github.com/vmware/vsphere-automation-sdk-go/runtime/log"
 	policyclient "github.com/vmware/vsphere-automation-sdk-go/runtime/protocol/client"
 	"github.com/vmware/vsphere-automation-sdk-go/runtime/security"
+	segSDK "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/segments"
+	segPortSDK "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/segments/ports"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/search"
 	"gopkg.in/ini.v1"
@@ -167,19 +166,23 @@ func listAttachmentsByNodeExternalId(nsxClients *NsxClients, vmExternalId string
 	return attachment_ids, nil
 }
 
-func listPortsByAttachmentIds(nsxClients *NsxClients, attachmentIds sets.String) (*[]nsxtmgr.LogicalPort, error) {
-	var portList []nsxtmgr.LogicalPort
-	localVarOptionals := make(map[string]interface{})
+func listPortsByAttachmentIds(nsxClients *NsxClients, attachmentIds sets.String) (*[]model.SegmentPort, error) {
+	var portList []model.SegmentPort
+	connector := nsxClients.PolicyConnector
+	searchClient := search.NewQueryClient(connector)
 	for attachmentId := range attachmentIds {
-		localVarOptionals["attachmentId"] = attachmentId
-		nsxClient := nsxClients.ManagerClient
-		log.Info(fmt.Sprintf("Searching logical port for vif attachment %s", attachmentId))
-		lsps, _, err := nsxClient.LogicalSwitchingApi.ListLogicalPorts(nsxClient.Context, localVarOptionals)
+		log.Info(fmt.Sprintf("Searching segment port for vif attachment %s", attachmentId))
+		searchString := fmt.Sprintf("resource_type:SegmentPort AND attachment.id:%s", attachmentId)
+		searchOp, err := searchClient.List(searchString, nil, nil, nil, nil, nil)
 		if err != nil {
 			return nil, err
 		}
-		for _, lsp := range lsps.Results {
-			portList = append(portList, lsp)
+		for _, obj := range searchOp.Results {
+			segPort, err := operatortypes.CastToBindingType[model.SegmentPort](obj, model.SegmentPortBindingType())
+			if err != nil {
+				return nil, err
+			}
+			portList = append(portList, segPort)
 		}
 	}
 	if len(portList) == 0 {
@@ -197,23 +200,28 @@ func inSlice(str string, s []string) bool {
 	return false
 }
 
-func filterPortsByNodeAddresses(nsxClients *NsxClients, ports *[]nsxtmgr.LogicalPort, nodeAddresses []string) ([]*nsxtmgr.LogicalPort, error) {
-	var filteredPorts []*nsxtmgr.LogicalPort
+func filterPortsByNodeAddresses(nsxClients *NsxClients, ports *[]model.SegmentPort, nodeAddresses []string) ([]*model.SegmentPort, error) {
+	var filteredPorts []*model.SegmentPort
 	log.Info(fmt.Sprintf("Found %d ports for node, checking addresses %v", len(*ports), nodeAddresses))
-	nsxClient := nsxClients.ManagerClient
+	connector := nsxClients.PolicyConnector
+	segPortStateClient := segPortSDK.NewStateClient(connector)
 	for _, port := range *ports {
-		logicalPort, _, err := nsxClient.LogicalSwitchingApi.GetLogicalPortState(nsxClient.Context, port.Id)
+		segId, err := operatortypes.ExtractSegmentIdFromPath(*port.ParentPath)
 		if err != nil {
 			return filteredPorts, err
 		}
-		if len(logicalPort.RealizedBindings) == 0 {
+		portState, err := segPortStateClient.Get(segId, *port.Id, nil, nil)
+		if err != nil {
+			return filteredPorts, err
+		}
+		if len(portState.RealizedBindings) == 0 {
 			continue
 		}
 		var collectedAddresses []string
-		for _, realizedBinding := range logicalPort.RealizedBindings {
-			address := realizedBinding.Binding.IpAddress
+		for _, realizedBinding := range portState.RealizedBindings {
+			address := *realizedBinding.Binding.IpAddress
 			if inSlice(address, nodeAddresses) && !inSlice(address, collectedAddresses) {
-				log.Info(fmt.Sprintf("Node address %s matches port %s", address, port.Id))
+				log.Info(fmt.Sprintf("Node address %s matches port %s", address, *port.Id))
 				// The addresses in logicalPort.RealizedBindings may be duplicate so we use collectedAddresses to ensure the uniqueness in filteredPorts.
 				collectedAddresses = append(collectedAddresses, address)
 				filteredPorts = append(filteredPorts, &port)
@@ -230,69 +238,6 @@ func filterPortsByNodeAddresses(nsxClients *NsxClients, ports *[]nsxtmgr.Logical
 	return filteredPorts, err
 }
 
-func searchNodePortByVcNameAddress(nsxClients *NsxClients, nodeName string, nodeAddress string) (*model.SegmentPort, error) {
-	log.Info(fmt.Sprintf("Searching segment port for node %s", nodeName))
-	connector := nsxClients.PolicyConnector
-	searchClient := search.NewDefaultQueryClient(connector)
-	// The format of node segment port display_name:
-	//   <vmx file's parent directory name>/<node vSphere name>.vmx@<tn-id>
-	// The vmx file's parent directory name can include VM name or a uid string for a vSAN VM
-	searchString := fmt.Sprintf("resource_type:SegmentPort AND display_name:*\\/%s.vmx*", nodeName)
-	ports, err := searchClient.List(searchString, nil, nil, nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	if len(ports.Results) == 0 {
-		return nil, errors.Errorf("segment port for node %s not found", nodeName)
-	}
-	portIndex := 0
-	portIndex, err = filterSegmentPorts(nsxClients, ports.Results, nodeName, nodeAddress)
-	if err != nil {
-		return nil, errors.Errorf("found %d segment ports for node %s, but none with address %s: %s", len(ports.Results), nodeName, nodeAddress, err)
-	}
-	portId, err := ports.Results[portIndex].Field("id")
-	if err != nil {
-		return nil, err
-	}
-	portPath, err := ports.Results[portIndex].Field("parent_path")
-	if err != nil {
-		return nil, err
-	}
-	portIdValue := (portId).(*data.StringValue).Value()
-	portPathValue := (portPath).(*data.StringValue).Value()
-	segmentPort := model.SegmentPort{
-		Id:   &portIdValue,
-		Path: &portPathValue,
-	}
-	return &segmentPort, nil
-}
-
-func filterSegmentPorts(nsxClients *NsxClients, ports []*data.StructValue, nodeName string, nodeAddress string) (int, error) {
-	log.Info(fmt.Sprintf("Found %d segment ports for node %s, checking addresses", len(ports), nodeName))
-	for idx, port := range ports {
-		portPolicyId, err := port.Field("id")
-		if err != nil {
-			return -1, err
-		}
-		portPolicyIdValue := (portPolicyId).(*data.StringValue).Value()
-		// there's an assumption that the policy ID has format "default:<manager_id>"
-		portMgrId := string([]byte(portPolicyIdValue)[8:])
-		nsxClient := nsxClients.ManagerClient
-		logicalPort, _, err := nsxClient.LogicalSwitchingApi.GetLogicalPortState(nsxClient.Context, portMgrId)
-		if err != nil {
-			return -1, err
-		}
-		if len(logicalPort.RealizedBindings) == 0 {
-			continue
-		}
-		address := logicalPort.RealizedBindings[0].Binding.IpAddress
-		if address == nodeAddress {
-			return idx, nil
-		}
-	}
-	return -1, errors.Errorf("no port matches")
-}
-
 func getConnectorTLSConfig(insecure bool, clientCertFile string, clientKeyFile string, caFile string) (*tls.Config, error) {
 	tlsConfig := tls.Config{InsecureSkipVerify: insecure}
 
@@ -583,9 +528,9 @@ func (r *ReconcileNode) Reconcile(request reconcile.Request) (reconcile.Result,
 		foundNodeTag := false
 		foundClusterTag := false
 		for _, tag := range lsp.Tags {
-			if tag.Scope == nodeNameScope && tag.Tag == request.Name {
+			if *tag.Scope == nodeNameScope && *tag.Tag == request.Name {
 				foundNodeTag = true
-			} else if tag.Scope == clusterScope && tag.Tag == cluster {
+			} else if *tag.Scope == clusterScope && *tag.Tag == cluster {
 				foundClusterTag = true
 			}
 		}
@@ -595,15 +540,26 @@ func (r *ReconcileNode) Reconcile(request reconcile.Request) (reconcile.Result,
 		}
 		reqLogger.Info("Updating node tag for port", "port.Id", lsp.Id)
 		if foundNodeTag == false {
-			var nodeTag = common.Tag{Scope: nodeNameScope, Tag: request.Name}
+			var nodeTag = model.Tag{Scope: &nodeNameScope, Tag: &request.Name}
 			lsp.Tags = append(lsp.Tags, nodeTag)
 		}
 		if foundClusterTag == false {
-			var clusterTag = common.Tag{Scope: clusterScope, Tag: cluster}
+			var clusterTag = model.Tag{Scope: &clusterScope, Tag: &cluster}
 			lsp.Tags = append(lsp.Tags, clusterTag)
 		}
-		nsxClient := nsxClients.ManagerClient
-		_, _, err = nsxClient.LogicalSwitchingApi.UpdateLogicalPort(nsxClient.Context, lsp.Id, *lsp)
+		connector := nsxClients.PolicyConnector
+		segPortStateClient := segSDK.NewPortsClient(connector)
+		segId, err := operatortypes.ExtractSegmentIdFromPath(*lsp.ParentPath)
+		if err != nil {
+			cachedNodeSet[nodeName] = &statusmanager.NodeStatus{
+				Addresses: nodeAddresses,
+				Success:   false,
+				Reason:    fmt.Sprintf("Error while achieving port with specific address for node %s: %v", nodeName, err),
+			}
+			r.status.SetFromNodes(cachedNodeSet)
+			return reconcile.Result{}, err
+		}
+		_, err = segPortStateClient.Update(segId, *lsp.Id, *lsp)
 		anyUpdate = true
 		if err != nil {
 			cachedNodeSet[nodeName] = &statusmanager.NodeStatus{
@@ -614,7 +570,6 @@ func (r *ReconcileNode) Reconcile(request reconcile.Request) (reconcile.Result,
 			r.status.SetFromNodes(cachedNodeSet)
 			return reconcile.Result{}, err
 		}
-
 	}
 	cachedNodeSet[nodeName] = &statusmanager.NodeStatus{
 		Addresses: nodeAddresses,

pkg/types/utils.go

@@ -5,11 +5,14 @@ package types
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/bindings"
+	"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
 	appsv1 "k8s.io/api/apps/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -45,3 +48,25 @@ func identifyAndGetInstance(resName string) (runtime.Object, error) {
 	}
 	return nil, errors.Errorf("failed to identify instance for: %s", resName)
 }
+
+func CastToBindingType[T any](dataValue *data.StructValue, destBindingType bindings.BindingType) (T, error) {
+	converter := bindings.NewTypeConverter()
+	obj, errs := converter.ConvertToGolang(dataValue, destBindingType)
+	if len(errs) > 0 {
+		t, _ := obj.(T)
+		return t, errs[0]
+	}
+	t, ok := obj.(T)
+	if !ok {
+		return t, fmt.Errorf("cast to bind type failed: %v is not of type %T", obj, t)
+	}
+	return t, nil
+}
+
+func ExtractSegmentIdFromPath(segmentPath string) (string, error) {
+	segments := strings.Split(segmentPath, "/infra/segments/")
+	if len(segments) > 1 {
+		return segments[len(segments)-1], nil
+	}
+	return "", fmt.Errorf("unable to find the Segment ID from provided path: %s", segmentPath)
+}