Merge pull request #4 from wenqiq/main-upstream

Main upstream sync

build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml

@@ -77,11 +77,17 @@ spec:
                 maxItems: 2
                 minItems: 0
                 type: array
+                x-kubernetes-validations:
+                - message: Value is immutable
+                  rule: self == oldSelf
               ipv4SubnetSize:
                 description: Size of Subnet based upon estimated workload count.
                 maximum: 65536
                 minimum: 16
                 type: integer
+                x-kubernetes-validations:
+                - message: Value is immutable
+                  rule: self == oldSelf
             type: object
           status:
             description: SubnetStatus defines the observed state of Subnet.

build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml

@@ -75,6 +75,9 @@ spec:
                 maximum: 65536
                 minimum: 16
                 type: integer
+                x-kubernetes-validations:
+                - message: Value is immutable
+                  rule: self == oldSelf
             type: object
           status:
             description: SubnetSetStatus defines the observed state of SubnetSet.

cmd/main.go

@@ -35,6 +35,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnet"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/controllers/subnetset"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipblocksinfo"
 	nodeservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/node"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/staticroute"
 	subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet"
@@ -109,13 +110,14 @@ func StartNSXServiceAccountController(mgr ctrl.Manager, commonService common.Ser
 	go commonctl.GenericGarbageCollector(make(chan bool), common.GCInterval, nsxServiceAccountReconcile.CollectGarbage)
 }
 
-func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService) {
+func StartNetworkInfoController(mgr ctrl.Manager, vpcService *vpc.VPCService, ipblocksInfoService *ipblocksinfo.IPBlocksInfoService) {
 	networkInfoReconciler := &networkinfocontroller.NetworkInfoReconciler{
 		Client:   mgr.GetClient(),
 		Scheme:   mgr.GetScheme(),
 		Recorder: mgr.GetEventRecorderFor("networkinfo-controller"),
 	}
 	networkInfoReconciler.Service = vpcService
+	networkInfoReconciler.IPBlocksInfoService = ipblocksInfoService
 	if err := networkInfoReconciler.Start(mgr); err != nil {
 		log.Error(err, "failed to create networkinfo controller", "controller", "NetworkInfo")
 		os.Exit(1)
@@ -203,8 +205,9 @@ func startServiceController(mgr manager.Manager, nsxClient *nsx.Client) {
 			log.Error(err, "failed to initialize staticroute commonService", "controller", "StaticRoute")
 			os.Exit(1)
 		}
+		ipblocksInfoService := ipblocksinfo.InitializeIPBlocksInfoService(commonService)
 		// Start controllers which only supports VPC
-		StartNetworkInfoController(mgr, vpcService)
+		StartNetworkInfoController(mgr, vpcService, ipblocksInfoService)
 		StartNamespaceController(mgr, cf, vpcService)
 		// Start subnet/subnetset controller.
 		if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil {

go.mod

@@ -6,10 +6,10 @@ replace (
 	github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis
 	github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1 => ./pkg/apis/vpc/v1alpha1
 	github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client
-	github.com/vmware/vsphere-automation-sdk-go/lib => github.com/zhengxiexie/vsphere-automation-sdk-go/lib v0.7.3-0.20240808083204-389375dc2535
-	github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/zhengxiexie/vsphere-automation-sdk-go/runtime v0.7.3-0.20240808083204-389375dc2535
-	github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/zhengxiexie/vsphere-automation-sdk-go/services/nsxt v0.12.3-0.20240808083204-389375dc2535
-	github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/zhengxiexie/vsphere-automation-sdk-go/services/nsxt-mp v0.6.3-0.20240808083204-389375dc2535
+	github.com/vmware/vsphere-automation-sdk-go/lib => github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a
+	github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a
+	github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a
+	github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a
 )
 
 require (

go.sum

@@ -143,17 +143,17 @@ github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wf
 github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=
 github.com/vmware/govmomi v0.27.4/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o=
 github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk=
+github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a h1:nF3PigKL+lN4ECHkgVJIZgLbpLrV6U6wkKHnIHOU9kA=
+github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:ysW7/EqFugBY2TcbvlDeRGaYIoG7Cs0i4l4WsMI/RmQ=
+github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a h1:b08LCEgSR6GSsvQzx2fxVbEXSKRnaGcMUqKjlgwR6xM=
+github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:DzLetYAmw1+vj7bqElRWEpuy40WYE/woL3alsymYa/c=
+github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a h1:XEgprSLuSKIxr7OPEzBWrlo39ra7pDaWFwAIjK0VV7s=
+github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:aJtyfDKvGyuP1ieRHCLoYjo2XtNZ401XfS7lCd43Bqs=
+github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a h1:4FmesihC1B7udmKl7B2giLydxibViMdyldSforV5qbU=
+github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:FX8UiCgNEOxweA73VZsyKZvMLPFfc70GBc1d4dj0nXI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/zhengxiexie/vsphere-automation-sdk-go/lib v0.7.3-0.20240808083204-389375dc2535 h1:RyRlKXlQBsDuuhi9xNdW5ygh7P9LPD4VwohMFFktpNI=
-github.com/zhengxiexie/vsphere-automation-sdk-go/lib v0.7.3-0.20240808083204-389375dc2535/go.mod h1:AEp9XK68cVmyxUtAYZekGBEauHwmMKCykLWOWkX7GII=
-github.com/zhengxiexie/vsphere-automation-sdk-go/runtime v0.7.3-0.20240808083204-389375dc2535 h1:ACBksAqrTPfO38jfPYUYkPxLEmkRb1kAnaiDbn2b9mI=
-github.com/zhengxiexie/vsphere-automation-sdk-go/runtime v0.7.3-0.20240808083204-389375dc2535/go.mod h1:DzLetYAmw1+vj7bqElRWEpuy40WYE/woL3alsymYa/c=
-github.com/zhengxiexie/vsphere-automation-sdk-go/services/nsxt v0.12.3-0.20240808083204-389375dc2535 h1:VIVbs8Vg/V5ntux2RFOAUw7p0p1BQVKulcoejvq0U54=
-github.com/zhengxiexie/vsphere-automation-sdk-go/services/nsxt v0.12.3-0.20240808083204-389375dc2535/go.mod h1:ODLiES6L4kslLAgLcLHPJJo7j1ez08hyJDIH0RTDGl0=
-github.com/zhengxiexie/vsphere-automation-sdk-go/services/nsxt-mp v0.6.3-0.20240808083204-389375dc2535 h1:MywC6dY0e3FAauytiZkTZ5myAhgTy0piytXBpGr+VGU=
-github.com/zhengxiexie/vsphere-automation-sdk-go/services/nsxt-mp v0.6.3-0.20240808083204-389375dc2535/go.mod h1:qcPRwB7KlQRpGFZs0HDzqOxXljIl2Szh7SPRvKUj0Ww=
 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
 go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=

pkg/apis/vpc/v1alpha1/subnet_types.go

@@ -20,6 +20,7 @@ type SubnetSpec struct {
 	// Size of Subnet based upon estimated workload count.
 	// +kubebuilder:validation:Maximum:=65536
 	// +kubebuilder:validation:Minimum:=16
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
 	// +kubebuilder:validation:Enum=Private;Public;PrivateTGW
@@ -28,6 +29,7 @@ type SubnetSpec struct {
 	// Subnet CIDRS.
 	// +kubebuilder:validation:MinItems=0
 	// +kubebuilder:validation:MaxItems=2
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
 	IPAddresses []string `json:"ipAddresses,omitempty"`
 	// DHCPConfig DHCP configuration.
 	DHCPConfig DHCPConfig `json:"DHCPConfig,omitempty"`

pkg/apis/vpc/v1alpha1/subnetset_types.go

@@ -12,6 +12,7 @@ type SubnetSetSpec struct {
 	// Size of Subnet based upon estimated workload count.
 	// +kubebuilder:validation:Maximum:=65536
 	// +kubebuilder:validation:Minimum:=16
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
 	IPv4SubnetSize int `json:"ipv4SubnetSize,omitempty"`
 	// Access mode of Subnet, accessible only from within VPC or from outside VPC.
 	// +kubebuilder:validation:Enum=Private;Public;PrivateTGW

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -25,6 +25,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/metrics"
 	_ "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
 	commonservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/ipblocksinfo"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
 )
 
@@ -36,10 +37,11 @@ var (
 // NetworkInfoReconciler NetworkInfoReconcile reconciles a NetworkInfo object
 // Actually it is more like a shell, which is used to manage nsx VPC
 type NetworkInfoReconciler struct {
-	Client   client.Client
-	Scheme   *apimachineryruntime.Scheme
-	Service  *vpc.VPCService
-	Recorder record.EventRecorder
+	Client              client.Client
+	Scheme              *apimachineryruntime.Scheme
+	Service             *vpc.VPCService
+	IPBlocksInfoService *ipblocksinfo.IPBlocksInfoService
+	Recorder            record.EventRecorder
 }
 
 func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -92,24 +94,24 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			return common.ResultRequeueAfter10sec, err
 		}
 
-		reason := ""
+		gatewayConnectionReason := ""
 		if !gatewayConnectionReady {
 			if ncName == commonservice.SystemVPCNetworkConfigurationName {
-				gatewayConnectionReady, reason, err = r.Service.ValidateGatewayConnectionStatus(&nc)
-				log.Info("got the gateway connection status", "gatewayConnectionReady", gatewayConnectionReady, "reason", reason)
+				gatewayConnectionReady, gatewayConnectionReason, err = r.Service.ValidateGatewayConnectionStatus(&nc)
+				log.Info("got the gateway connection status", "gatewayConnectionReady", gatewayConnectionReady, "gatewayConnectionReason", gatewayConnectionReason)
 				if err != nil {
 					log.Error(err, "failed to validate the edge and gateway connection", "org", nc.Org, "project", nc.NSXProject)
 					updateFail(r, ctx, obj, &err, r.Client, nil)
 					return common.ResultRequeueAfter10sec, err
 				}
-				setVPCNetworkConfigurationStatusWithGatewayConnection(ctx, r.Client, vpcNetworkConfiguration, gatewayConnectionReady, reason)
+				setVPCNetworkConfigurationStatusWithGatewayConnection(ctx, r.Client, vpcNetworkConfiguration, gatewayConnectionReady, gatewayConnectionReason)
 			} else {
 				log.Info("skipping reconciling the network info because the system gateway connection is not ready", "NetworkInfo", req.NamespacedName)
 				return common.ResultRequeueAfter60sec, nil
 			}
 		}
-
-		createdVpc, err := r.Service.CreateOrUpdateVPC(obj, &nc)
+		lbProvider := r.Service.GetLBProvider()
+		createdVpc, err := r.Service.CreateOrUpdateVPC(obj, &nc, lbProvider)
 		if err != nil {
 			log.Error(err, "create vpc failed, would retry exponentially", "VPC", req.NamespacedName)
 			updateFail(r, ctx, obj, &err, r.Client, nil)
@@ -118,9 +120,21 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 		var privateIPs []string
 		var vpcConnectivityProfilePath string
-		if vpc.IsPreCreatedVPC(nc) {
+		var nsxLBSPath string
+		isPreCreatedVPC := vpc.IsPreCreatedVPC(nc)
+		if isPreCreatedVPC {
 			privateIPs = createdVpc.PrivateIps
 			vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile
+			// Retrieve NSX lbs path if Avi is not used with the pre-created VPC.
+			if createdVpc.LoadBalancerVpcEndpoint == nil || createdVpc.LoadBalancerVpcEndpoint.Enabled == nil ||
+				!*createdVpc.LoadBalancerVpcEndpoint.Enabled {
+				nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(*createdVpc.Path)
+				if err != nil {
+					log.Error(err, "failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path)
+					updateFail(r, ctx, obj, &err, r.Client, nil)
+					return common.ResultRequeueAfter10sec, err
+				}
+			}
 		} else {
 			privateIPs = nc.PrivateIPs
 			vpcConnectivityProfilePath = nc.VPCConnectivityProfile
@@ -134,30 +148,17 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			updateFail(r, ctx, obj, &err, r.Client, nil)
 			return common.ResultRequeueAfter10sec, err
 		}
-
+		hasExternalIPs := true
 		if ncName == commonservice.SystemVPCNetworkConfigurationName {
 			if len(vpcConnectivityProfile.ExternalIpBlocks) == 0 {
-				setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, false)
+				hasExternalIPs = false
 				log.Error(err, "there is no ExternalIPBlock in VPC ConnectivityProfile", "VPC", req.NamespacedName)
-			} else {
-				setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, true)
 			}
-		}
-		isEnableAutoSNAT := func() bool {
-			if vpcConnectivityProfile.ServiceGateway == nil || vpcConnectivityProfile.ServiceGateway.Enable == nil {
-				return false
-			}
-			if *vpcConnectivityProfile.ServiceGateway.Enable {
-				if vpcConnectivityProfile.ServiceGateway.NatConfig == nil || vpcConnectivityProfile.ServiceGateway.NatConfig.EnableDefaultSnat == nil {
-					return false
-				}
-				return *vpcConnectivityProfile.ServiceGateway.NatConfig.EnableDefaultSnat
-			}
-			return false
+			setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, hasExternalIPs)
 		}
 		// currently, auto snat is not exposed, and use default value True
 		// checking autosnat to support future extension in vpc configuration
-		autoSnatEnabled := isEnableAutoSNAT()
+		autoSnatEnabled := r.Service.IsEnableAutoSNAT(vpcConnectivityProfile)
 		if autoSnatEnabled {
 			snatIP, err = r.Service.GetDefaultSNATIP(*createdVpc)
 			if err != nil {
@@ -180,19 +181,14 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 				updateFail(r, ctx, obj, &err, r.Client, nil)
 				return common.ResultRequeueAfter10sec, err
 			}
-			if autoSnatEnabled {
-				log.Info("detected that the AutoSnat is enabled", "req", req.NamespacedName)
-				setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, true)
-			} else {
-				log.Info("detected that the AutoSnat is disabled", "req", req.NamespacedName)
-				setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, false)
-			}
+			log.Info("got the AutoSnat status", "autoSnatEnabled", autoSnatEnabled, "req", req.NamespacedName)
+			setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, autoSnatEnabled)
 		}
 
 		// if lb vpc enabled, read avi subnet path and cidr
 		// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
 		// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
-		if !r.Service.NSXLBEnabled() && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
+		if lbProvider == vpc.AVILB && createdVpc.LoadBalancerVpcEndpoint != nil && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
 			path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
 			if err != nil {
 				log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
@@ -214,9 +210,17 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			PrivateIPs:              privateIPs,
 			VPCPath:                 *createdVpc.Path,
 		}
+
+		if !isPreCreatedVPC {
+			nsxLBSPath = r.Service.GetDefaultNSXLBSPathByVPC(*createdVpc.Id)
+		}
 		// AKO needs to know the AVI subnet path created by NSX
-		setVPCNetworkConfigurationStatusWithLBS(ctx, r.Client, ncName, state.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id), *createdVpc.Path)
+		setVPCNetworkConfigurationStatusWithLBS(ctx, r.Client, ncName, state.Name, path, nsxLBSPath, *createdVpc.Path)
 		updateSuccess(r, ctx, obj, r.Client, state, nc.Name, path)
+		if ncName == commonservice.SystemVPCNetworkConfigurationName && (!gatewayConnectionReady || !autoSnatEnabled || !hasExternalIPs) {
+			log.Info("requeuing the NetworkInfo CR because VPCNetworkConfiguration system is not ready", "gatewayConnectionReason", gatewayConnectionReason, "autoSnatEnabled", autoSnatEnabled, "hasExternalIPs", hasExternalIPs, "req", req)
+			return common.ResultRequeueAfter60sec, nil
+		}
 	} else {
 		if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
 			metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)
@@ -274,13 +278,15 @@ func (r *NetworkInfoReconciler) setupWithManager(mgr ctrl.Manager) error {
 				MaxConcurrentReconciles: common.NumReconcile(),
 			}).
 		Watches(
-			// For created/removed network config, add/remove from vpc network config cache.
+			// For created/removed network config, add/remove from vpc network config cache,
+			// and update IPBlocksInfo.
 			// For modified network config, currently only support appending ips to public ip blocks,
 			// update network config in cache and update nsx vpc object.
 			&v1alpha1.VPCNetworkConfiguration{},
 			&VPCNetworkConfigurationHandler{
-				Client:     mgr.GetClient(),
-				vpcService: r.Service,
+				Client:              mgr.GetClient(),
+				vpcService:          r.Service,
+				ipBlocksInfoService: r.IPBlocksInfoService,
 			},
 			builder.WithPredicates(VPCNetworkConfigurationPredicate)).
 		Complete(r)