add CRUD logic in antreaConfig reconciler

Signed-off-by: Bin Liu <[email protected]>

addons/controllers/antrea/antreaconfig_controller.go

@@ -7,13 +7,18 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/go-logr/logr"
 	yaml "gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	vsphere "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
 	clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	clusterapiutil "sigs.k8s.io/cluster-api/util"
 	clusterapipatchutil "sigs.k8s.io/cluster-api/util/patch"
@@ -24,19 +29,30 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	nsxoperatorapi "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
+
 	addonconfig "github.com/vmware-tanzu/tanzu-framework/addons/pkg/config"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
 	"github.com/vmware-tanzu/tanzu-framework/addons/predicates"
 	cniv1alpha1 "github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs/cni/v1alpha1"
 )
 
+const (
+	defaultTargetNameSpace = "vmware-system-antrea"
+	defaultSecretName      = "supervisor-cred"
+	defaultAPIGroup        = "nsx.vmware.com"
+	defaultResource        = "nsxserviceaccounts"
+	clusterNameLabel       = "tkg.tanzu.vmware.com/cluster-name"
+)
+
 // AntreaConfigReconciler reconciles a AntreaConfig object
 type AntreaConfigReconciler struct {
 	client.Client
-	Log    logr.Logger
-	Scheme *runtime.Scheme
-	Config addonconfig.AntreaConfigControllerConfig
+	nsxOperatorClient client.Client
+	Log               logr.Logger
+	Scheme            *runtime.Scheme
+	Config            addonconfig.AntreaConfigControllerConfig
 }
 
 // +kubebuilder:rbac:groups=addons.tanzu.vmware.com,resources=antreaconfigs,verbs=get;list;watch;create;update;patch;delete
@@ -133,6 +149,7 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfig(
 
 	// If AntreaConfig is marked for deletion, then no reconciliation is needed
 	if !antreaConfig.GetDeletionTimestamp().IsZero() {
+		r.deleteAccounts(antreaConfig)
 		return ctrl.Result{}, nil
 	}
 
@@ -170,10 +187,138 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfigNormal(
 		return err
 	}
 
+	if antreaConfig.Spec.AntreaNsx.AntreaNsxProvider != nil && antreaConfig.Spec.AntreaNsx.AntreaNsxInline != nil {
+		err := fmt.Errorf("AntreaNsxProvider can not be used with AntreaNsxInline in antreaConfig")
+		antreaConfig.Status.Message = err.Error()
+	}
 	// update status.secretRef
 	dataValueSecretName := util.GenerateDataValueSecretName(cluster.Name, constants.AntreaAddonName)
 	antreaConfig.Status.SecretRef = dataValueSecretName
 
+	err := r.confirmProviderServiceAccount(antreaConfig)
+	if err != nil {
+		return err
+	}
+	err = r.confirmNsxServiceAccount(antreaConfig)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getClusterName(antreaConfig *cniv1alpha1.AntreaConfig) (name string, exists bool) {
+	name, exists = antreaConfig.Labels[clusterNameLabel]
+	if !exists {
+		index := strings.Index(antreaConfig.Name, "-antrea-package")
+		if index > 0 {
+			name = antreaConfig.Name[:index]
+			exists = true
+		}
+	}
+	return
+}
+
+func (r *AntreaConfigReconciler) confirmNsxServiceAccount(antreaConfig *cniv1alpha1.AntreaConfig) error {
+	account := &nsxoperatorapi.NSXServiceAccount{}
+
+	clusterName, exists := getClusterName(antreaConfig)
+	if !exists {
+		return fmt.Errorf("invalid antreaConfig Name")
+	}
+	account.Name = fmt.Sprintf("%s-%s-antrea-nsx", antreaConfig.Namespace, clusterName)
+	account.Namespace = antreaConfig.Namespace
+	account.OwnerReferences = []metav1.OwnerReference{
+		{
+			APIVersion: defaultAPIGroup,
+			Kind:       "Cluster",
+			Name:       clusterName,
+		},
+	}
+
+	err := r.Client.Get(context.TODO(), types.NamespacedName{
+		Namespace: account.Namespace,
+		Name:      account.Name,
+	}, account)
+	if err == nil {
+		r.Log.Info("NSXServiceAccount %s/%s already exists", account.Namespace, account.Name)
+		return nil
+	}
+	if err != nil && apierrors.IsNotFound(err) {
+		r.Log.Info("failed to get NSXServiceAccount %s/%s", account.Namespace, account.Name)
+		return err
+	}
+
+	result, err := controllerutil.CreateOrPatch(context.TODO(), r.Client, account, nil)
+	if err != nil {
+		r.Log.Error(err, "Error creating or patching NSXServiceAccount", account.Namespace, account.Name)
+	} else {
+		r.Log.Info(fmt.Sprintf("NSXServiceAccount %s/%s created %s", account.Namespace, account.Name, result))
+	}
+	return err
+}
+
+func (r *AntreaConfigReconciler) confirmProviderServiceAccount(antreaConfig *cniv1alpha1.AntreaConfig) error {
+	provider := &vsphere.ProviderServiceAccount{}
+	clusterName, exists := getClusterName(antreaConfig)
+	if !exists {
+		return fmt.Errorf("invalid antreaConfig Name")
+	}
+	provider.Name = fmt.Sprintf("%s-%s-antrea-nsx", antreaConfig.Namespace, clusterName)
+	provider.Namespace = antreaConfig.Namespace
+
+	provider.Spec = vsphere.ProviderServiceAccountSpec{
+		Ref: &corev1.ObjectReference{
+			Name:      provider.Name,
+			Namespace: provider.Namespace,
+		},
+		TargetNamespace:  defaultTargetNameSpace,
+		TargetSecretName: defaultSecretName,
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{defaultAPIGroup},
+				Resources:     []string{defaultResource},
+				ResourceNames: []string{clusterName},
+				Verbs:         []string{"get", "list", "watch"},
+			},
+			{
+				APIGroups:     []string{""},
+				Resources:     []string{"secrets"},
+				ResourceNames: []string{fmt.Sprintf("%s-%s-nsx-cert", antreaConfig.Namespace, clusterName)},
+				Verbs:         []string{"get", "list", "watch"},
+			},
+		},
+	}
+	result, err := controllerutil.CreateOrPatch(context.TODO(), r.Client, provider, nil)
+	if err != nil {
+		r.Log.Error(err, "Error creating or patching ProviderServiceAccount", provider.Namespace, provider.Name)
+	} else {
+		r.Log.Info(fmt.Sprintf("ProviderServiceAccount %s/%s created %s", provider.Namespace, provider.Name, result))
+	}
+	return err
+}
+
+func (r *AntreaConfigReconciler) deleteAccounts(antreaConfig *cniv1alpha1.AntreaConfig) error {
+	account := &nsxoperatorapi.NSXServiceAccount{}
+	clusterName, exists := getClusterName(antreaConfig)
+	if !exists {
+		return fmt.Errorf("invalid antreaConfig Name")
+	}
+	account.Name = fmt.Sprintf("%s-%s-antrea-nsx", antreaConfig.Namespace, clusterName)
+	account.Namespace = antreaConfig.Namespace
+	err := r.Client.Delete(context.TODO(), account)
+	if err != nil {
+		r.Log.Error(err, "failed to delete NSXServiceAccount", account.Namespace, account.Name)
+		return err
+	}
+
+	provider := &vsphere.ProviderServiceAccount{}
+	provider.Name = fmt.Sprintf("%s-%s-antrea-nsx", antreaConfig.Namespace, clusterName)
+	provider.Namespace = antreaConfig.Namespace
+	err = r.Client.Delete(context.TODO(), provider)
+	if err != nil {
+		r.Log.Error(err, "failed to delete ProviderServiceAccount", provider.Namespace, provider.Name)
+		return err
+	}
 	return nil
 }
 

addons/controllers/antrea/antreaconfig_util.go

@@ -31,16 +31,16 @@ type antrea struct {
 }
 
 type antreaNsx struct {
-	Enable            bool              `yaml:"enable,omitempty"`
-	BootstrapFrom     string            `yaml:"bootstrapFrom,omitempty"`
-	AntreaNsxProvider antreaNsxProvider `yaml:"provider,omitempty"`
-	AntreaNsxInline   antreaNsxInline   `yaml:"inline,omitempty"`
-	AntreaNsxConfig   antreaNsxConfig   `yaml:"config,omitempty"`
+	Enable            bool               `yaml:"enable,omitempty"`
+	BootstrapFrom     string             `yaml:"bootstrapFrom,omitempty"`
+	AntreaNsxProvider *antreaNsxProvider `yaml:"provider,omitempty"`
+	AntreaNsxInline   *antreaNsxInline   `yaml:"inline,omitempty"`
+	AntreaNsxConfig   antreaNsxConfig    `yaml:"config,omitempty"`
 }
 
 type antreaNsxProvider struct {
 	ApiGroup string `yaml:"apiGroup,omitempty"`
-	kind     string `yaml:"kind,omitempty"`
+	Kind     string `yaml:"kind,omitempty"`
 }
 
 type antreaNsxInline struct {
@@ -217,5 +217,22 @@ func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.An
 	configSpec.Antrea.AntreaConfigDataValue.FeatureGates.ServiceExternalIP = config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.ServiceExternalIP
 	configSpec.Antrea.AntreaConfigDataValue.FeatureGates.Multicast = config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.Multicast
 
+	//nsx config
+	if config.Spec.AntreaNsx.Enable {
+		configSpec.AntreaNsx.Enable = config.Spec.AntreaNsx.Enable
+		if config.Spec.AntreaNsx.AntreaNsxProvider != nil {
+			configSpec.AntreaNsx.AntreaNsxProvider.ApiGroup = config.Spec.AntreaNsx.AntreaNsxProvider.ApiGroup
+			configSpec.AntreaNsx.AntreaNsxProvider.Kind = config.Spec.AntreaNsx.AntreaNsxProvider.Kind
+		} else if config.Spec.AntreaNsx.AntreaNsxInline == nil {
+			configSpec.AntreaNsx.AntreaNsxProvider.ApiGroup = "nsx.vmware.com"
+			configSpec.AntreaNsx.AntreaNsxProvider.Kind = "NSXServiceAccount"
+		}
+		if config.Spec.AntreaNsx.AntreaNsxInline != nil {
+			configSpec.AntreaNsx.AntreaNsxInline.NsxManagers = config.Spec.AntreaNsx.AntreaNsxInline.NsxManagers
+			configSpec.AntreaNsx.AntreaNsxInline.ClusterName = config.Spec.AntreaNsx.AntreaNsxInline.ClusterName
+			configSpec.AntreaNsx.AntreaNsxInline.NsxCertRef = config.Spec.AntreaNsx.AntreaNsxInline.NsxCertRef
+		}
+	}
+
 	return configSpec, nil
 }

addons/go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/vmware-tanzu/carvel-kapp-controller v0.35.0
 	github.com/vmware-tanzu/carvel-secretgen-controller v0.5.0
 	github.com/vmware-tanzu/carvel-vendir v0.26.0
+	github.com/vmware-tanzu/nsx-operator v0.0.0-20221031121446-ac06eba15212
 	github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs v0.0.0-20220907220230-c1137d344dd3
 	github.com/vmware-tanzu/tanzu-framework/apis/run v0.0.0-20220907220230-c1137d344dd3
 	github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20211202185235-43eb44c09ecd
@@ -90,7 +91,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/vmware-tanzu/net-operator-api v0.0.0-20210401185409-b0dc6c297707 // indirect
 	github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20211209213435-0f4ab286f64f // indirect
-	github.com/vmware/govmomi v0.27.1 // indirect
+	github.com/vmware/govmomi v0.27.4 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.22.0 // indirect

addons/go.sum

@@ -725,14 +725,16 @@ github.com/vmware-tanzu/carvel-vendir v0.26.0 h1:Q98tPnH9WUAWE2vJSAP0lsHGsfwGjfW
 github.com/vmware-tanzu/carvel-vendir v0.26.0/go.mod h1:JcuNNVONFbZTbm/GjtGiWUfFrt17YBQzeGT9+gY1+yY=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20210401185409-b0dc6c297707 h1:2onys8tWlQh7DFiOz6+68AwJdW9EBOEv6RTKzwh1x7A=
 github.com/vmware-tanzu/net-operator-api v0.0.0-20210401185409-b0dc6c297707/go.mod h1:pDB0pUiFYufuP3lUkQX9fZ67PYnKvqBpDcJN3mSrw5U=
+github.com/vmware-tanzu/nsx-operator v0.0.0-20221031121446-ac06eba15212 h1:g+AN4Z7Zp14Ket33wfAPejEEvjh4kB6wjfrMyXo5wV4=
+github.com/vmware-tanzu/nsx-operator v0.0.0-20221031121446-ac06eba15212/go.mod h1:Exsg90TOJ0FQBFzmdYurJNVCh9PMN3IkAr/tA+rKrLQ=
 github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20211202185235-43eb44c09ecd h1:BXz4aAPzRAYD8x8LEhjEsmvTj9mCvesnr4ApT1Ay4YY=
 github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20211202185235-43eb44c09ecd/go.mod h1:mubK0QMyaA2TbeAmGsu2GVfiqDFppNUAUqoMPoKFgzM=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20211209213435-0f4ab286f64f h1:RUuS5lh25citvQoXmDSfxJ1BB72LXOjD5cXvJETJ7Cc=
 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20211209213435-0f4ab286f64f/go.mod h1:5rqRJ9zGR+KnKbkGx373WgN8xJpvAj99kHnfoDYRO5I=
 github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20211209213435-0f4ab286f64f h1:wwYUf16/g8bLywQMQJB5VHbDtuf6aOFH24Ar2/yA7+I=
 github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20211209213435-0f4ab286f64f/go.mod h1:dfYrWS8DMRN+XZfhu8M4LVHmeGvYB29Ipd7j4uIq+mU=
-github.com/vmware/govmomi v0.27.1 h1:Rf3o1btFrkJa9be5KtgJ4CyOO8mbFnBxmNtAVHNyFes=
-github.com/vmware/govmomi v0.27.1/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o=
+github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I=
+github.com/vmware/govmomi v0.27.4/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o=
 github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=

apis/addonconfigs/cni/v1alpha1/antreaconfig_types.go

@@ -194,22 +194,25 @@ type AntreaFeatureGates struct {
 
 // AntreaConfigStatus defines the observed state of AntreaConfig
 type AntreaConfigStatus struct {
+	// Message to indicate failure reason
+	// +kubebuilder:validation:Optional
+	Message string `json:"message,omitempty"`
 	// Reference to the data value secret created by controller
 	// +kubebuilder:validation:Optional
 	SecretRef string `json:"secretRef,omitempty"`
 }
 
 type AntreaNsx struct {
-	Enable            bool              `json:"enable,omitempty"`
-	BootstrapFrom     string            `json:"bootstrapFrom,omitempty"`
-	AntreaNsxProvider AntreaNsxProvider `json:"provider,omitempty"`
-	AntreaNsxInline   AntreaNsxInline   `json:"inline,omitempty"`
-	AntreaNsxConfig   AntreaNsxConfig   `json:"config,omitempty"`
+	Enable            bool               `json:"enable,omitempty"`
+	BootstrapFrom     string             `json:"bootstrapFrom,omitempty"`
+	AntreaNsxProvider *AntreaNsxProvider `json:"provider,omitempty"`
+	AntreaNsxInline   *AntreaNsxInline   `json:"inline,omitempty"`
+	AntreaNsxConfig   AntreaNsxConfig    `json:"config,omitempty"`
 }
 
 type AntreaNsxProvider struct {
 	ApiGroup string `json:"apiGroup,omitempty"`
-	kind     string `json:"kind,omitempty"`
+	Kind     string `json:"kind,omitempty"`
 }
 
 type AntreaNsxInline struct {

apis/addonconfigs/config/crd/bases/cni.tanzu.vmware.com_antreaconfigs.yaml

@@ -258,6 +258,9 @@ spec:
           status:
             description: AntreaConfigStatus defines the observed state of AntreaConfig
             properties:
+              message:
+                description: Message to indicate failure reason
+                type: string
               secretRef:
                 description: Reference to the data value secret created by controller
                 type: string

cli/runtime/generated/crd/bases/config.tanzu.vmware.com_clientconfigs.yaml

@@ -157,7 +157,8 @@ spec:
                       type: object
                     type: array
                   edition:
-                    description: Edition
+                    description: 'Deprecated: Edition has been deprecated and will
+                      be removed from future version Edition'
                     type: string
                   repositories:
                     description: Repositories are the plugin repositories.
@@ -395,8 +396,8 @@ spec:
               type: object
             type: array
           current:
-            description: 'CurrentServer in use. Note: Shall be deprecated in a future
-              version. Superseded by CurrentContext.'
+            description: CurrentServer in use. Deprecation targeted for a a future
+              version. Superseded by CurrentContext.
             type: string
           currentContext:
             additionalProperties:
@@ -411,11 +412,11 @@ spec:
           metadata:
             type: object
           servers:
-            description: 'KnownServers available. Note: Shall be deprecated in a future
-              version. Superseded by KnownContexts.'
+            description: KnownServers available. Deprecation targeted for a a future
+              version. Superseded by KnownContexts.
             items:
-              description: 'Server connection. Note: Shall be deprecated in a future
-                version. Superseded by Context.'
+              description: Server connection. Deprecation targeted for a a future
+                version. Superseded by Context.
               properties:
                 discoverySources:
                   description: DiscoverySources determines from where to discover