From b1e0aa2e97a1183dc8bd8888418a961aa29bf6a1 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Mon, 5 Dec 2022 11:43:13 +0800 Subject: [PATCH 1/2] extend antreaconfig crd to support antreaNsx This patch extends antreaConfig to support antreaNsx, once antreaNsx is enabled in antreaConfig, addon controller will 1. create nsxServiceAccount 2. create providerServiceAccount 3. update values.yaml in xxxx-antrea-package secret with antreaNsx enabled upon the workload cluster is deleted, addon controller will 1. delete nsxServiceAccount 2. delete providerServiceAccount 3. delete xxxx-antrea-package secret Signed-off-by: Bin Liu --- .../antrea/antreaconfig_controller.go | 210 +++++++++++++++++- .../controllers/antrea/antreaconfig_util.go | 67 +++++- .../antreaconfig_controller_test.go | 143 ++++++++++++ .../controllers/testdata/antrea-test-2.yaml | 96 ++++++++ addons/go.mod | 3 +- addons/go.sum | 6 +- addons/main.go | 3 + addons/pkg/constants/constants.go | 3 + .../cni/v1alpha1/antreaconfig_types.go | 57 +++++ .../cni/v1alpha1/zz_generated.deepcopy.go | 93 ++++++++ .../cni.tanzu.vmware.com_antreaconfigs.yaml | 57 +++++ .../cni.tanzu.vmware.com_antreaconfigs.yaml | 57 +++++ 12 files changed, 782 insertions(+), 13 deletions(-) create mode 100644 addons/controllers/testdata/antrea-test-2.yaml diff --git a/addons/controllers/antrea/antreaconfig_controller.go b/addons/controllers/antrea/antreaconfig_controller.go index 7d0d5ae48b..91f5ede6b8 100644 --- a/addons/controllers/antrea/antreaconfig_controller.go +++ b/addons/controllers/antrea/antreaconfig_controller.go @@ -7,13 +7,17 @@ package controllers import ( "context" "fmt" + "strings" "github.com/go-logr/logr" yaml "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + vsphere "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterapiutil "sigs.k8s.io/cluster-api/util" clusterapipatchutil "sigs.k8s.io/cluster-api/util/patch" @@ -24,6 +28,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/source" + nsxoperatorapi "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + cutil "github.com/vmware-tanzu/tanzu-framework/addons/controllers/utils" addonconfig "github.com/vmware-tanzu/tanzu-framework/addons/pkg/config" "github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants" @@ -32,6 +38,24 @@ import ( cniv1alpha1 "github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs/cni/v1alpha1" ) +const ( + antreaTargetNameSpace = "vmware-system-antrea" + antreaSecretName = "supervisor-cred" + nsxServiceAccountAPIGroup = "nsx.vmware.com" + nsxServiceAccountKind = "nsxserviceaccounts" + clusterNameLabel = "tkg.tanzu.vmware.com/cluster-name" +) + +// vsphereAntreaConfigProviderServiceAccountAggregatedClusterRole is the cluster role to assign permissions to capv provider +var vsphereAntreaConfigProviderServiceAccountAggregatedClusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.VsphereAntreaConfigProviderServiceAccountAggregatedClusterRole, + Labels: map[string]string{ + constants.CAPVClusterRoleAggregationRuleLabelSelectorKey: constants.CAPVClusterRoleAggregationRuleLabelSelectorValue, + }, + }, +} + // AntreaConfigReconciler reconciles a AntreaConfig object type AntreaConfigReconciler struct { client.Client @@ -42,6 +66,8 @@ type AntreaConfigReconciler struct { // +kubebuilder:rbac:groups=addons.tanzu.vmware.com,resources=antreaconfigs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=addons.tanzu.vmware.com,resources=antreaconfigs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=providerserviceaccounts,verbs=get;create;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=nsx.vmware.com,resources=nsxserviceaccounts,verbs=get;create;list;watch;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -130,6 +156,7 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfig( // If AntreaConfig is marked for deletion, then no reconciliation is needed if !antreaConfig.GetDeletionTimestamp().IsZero() { + r.deregisterAntreaNSX(ctx, antreaConfig, cluster) return ctrl.Result{}, nil } @@ -167,10 +194,191 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfigNormal( return err } + if antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef != nil && antreaConfig.Spec.AntreaNsx.BootstrapFrom.Inline != nil { + err := fmt.Errorf("providerRef and inline should not be both set in AntreaConfig.spec.antreaNsx.bootstrapFrom") + antreaConfig.Status.Message = err.Error() + } else { + // clear the message here. + antreaConfig.Status.Message = "" + } // update status.secretRef dataValueSecretName := util.GenerateDataValueSecretName(cluster.Name, constants.AntreaAddonName) antreaConfig.Status.SecretRef = dataValueSecretName + return r.registerAntreaNSX(ctx, antreaConfig, cluster) +} + +func getClusterName(antreaConfig *cniv1alpha1.AntreaConfig) (name string, exists bool) { + name, exists = antreaConfig.Labels[clusterNameLabel] + if !exists { + index := strings.Index(antreaConfig.Name, "-antrea-package") + if index > 0 { + name = antreaConfig.Name[:index] + exists = true + } + } + return +} + +func (r *AntreaConfigReconciler) getProviderServiceAccountName(clusterName string) string { + return fmt.Sprintf("%s-antrea", clusterName) +} + +func (r *AntreaConfigReconciler) getNSXServiceAccountName(clusterName string) string { + return fmt.Sprintf("%s-antrea", clusterName) +} + +func (r *AntreaConfigReconciler) ensureNsxServiceAccount(ctx context.Context, antreaConfig *cniv1alpha1.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error { + account := &nsxoperatorapi.NSXServiceAccount{} + + account.Name = r.getNSXServiceAccountName(cluster.Name) + account.Namespace = antreaConfig.Namespace + account.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: cluster.APIVersion, + Kind: cluster.Kind, + Name: cluster.Name, + UID: cluster.UID, + }, + } + + err := r.Client.Get(ctx, types.NamespacedName{ + Namespace: account.Namespace, + Name: account.Name, + }, account) + if err == nil { + r.Log.Info("NSXServiceAccount %s/%s already exists", account.Namespace, account.Name) + return nil + } + if err != nil && !apierrors.IsNotFound(err) { + r.Log.Info("Found no existing NSXServiceAccount %s/%s", account.Namespace, account.Name) + return err + } + + result, err := controllerutil.CreateOrPatch(ctx, r.Client, account, nil) + if err != nil { + r.Log.Error(err, "Error creating or patching NSXServiceAccount", account.Namespace, account.Name) + } else { + r.Log.Info(fmt.Sprintf("NSXServiceAccount %s/%s created %s", account.Namespace, account.Name, result)) + } + return err +} + +func (r *AntreaConfigReconciler) ensureProviderServiceAccount(ctx context.Context, antreaConfig *cniv1alpha1.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error { + provider := &vsphere.ProviderServiceAccount{} + vsphereCluster, err := cutil.VSphereClusterParavirtualForCAPICluster(ctx, r.Client, cluster) + if err != nil { + return err + } + clusterName, _ := getClusterName(antreaConfig) + nsxSAName := clusterName + "-antrea" + nsxSecretName := clusterName + "-antrea-nsx-cert" + clusterName = vsphereCluster.Name + providerServiceAccountRBACRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{nsxServiceAccountAPIGroup}, + Resources: []string{nsxServiceAccountKind}, + ResourceNames: []string{nsxSAName}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + ResourceNames: []string{fmt.Sprintf(nsxSecretName)}, + Verbs: []string{"get", "list", "watch"}, + }, + } + _, err = controllerutil.CreateOrPatch(ctx, r.Client, vsphereAntreaConfigProviderServiceAccountAggregatedClusterRole, func() error { + vsphereAntreaConfigProviderServiceAccountAggregatedClusterRole.Rules = providerServiceAccountRBACRules + return nil + }) + if err != nil { + r.Log.Error(err, "Error creating or patching cluster role", "name", vsphereAntreaConfigProviderServiceAccountAggregatedClusterRole) + return err + } + provider.Name = r.getProviderServiceAccountName(clusterName) + provider.Namespace = antreaConfig.Namespace + provider.Spec = vsphere.ProviderServiceAccountSpec{ + Ref: &corev1.ObjectReference{ + APIVersion: cluster.APIVersion, + Kind: cluster.Kind, + Name: clusterName, + UID: cluster.UID, + }, + TargetNamespace: antreaTargetNameSpace, + TargetSecretName: antreaSecretName, + Rules: providerServiceAccountRBACRules, + } + result, err := controllerutil.CreateOrPatch(ctx, r.Client, provider, func() error { + return controllerutil.SetControllerReference(vsphereCluster, provider, r.Scheme) + }) + if err != nil { + r.Log.Error(err, "Error creating or patching ProviderServiceAccount", provider.Namespace, provider.Name) + } else { + r.Log.Info(fmt.Sprintf("ProviderServiceAccount %s/%s created %s： %+v", provider.Namespace, provider.Name, result, provider)) + } + return err +} + +func (r *AntreaConfigReconciler) registerAntreaNSX(ctx context.Context, antreaConfig *cniv1alpha1.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error { + if !antreaConfig.Spec.AntreaNsx.Enable || antreaConfig.Spec.AntreaNsx.BootstrapFrom.Inline != nil { + r.Log.Info("antreaNsx is not enabled or inline is set, there is no ProviderServiceAccount or NsxServiceAccount to be created") + r.deregisterAntreaNSX(ctx, antreaConfig, cluster) + return nil + } + if antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef != nil { + if strings.ToLower(antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.Kind) != nsxServiceAccountKind || + strings.ToLower(antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.ApiGroup) != nsxServiceAccountAPIGroup { + err := fmt.Errorf("either ProviderRef.Kind(%s) or ProviderRef.ApiGroup(%s) is invalid, expcted:ProviderRef.Kind(%s) ProviderRef.ApiGroup(%s)", + antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.Kind, antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.ApiGroup, + nsxServiceAccountKind, nsxServiceAccountAPIGroup) + antreaConfig.Status.Message = err.Error() + return err + } + } + antreaConfig.Status.Message = "" + err := r.ensureProviderServiceAccount(ctx, antreaConfig, cluster) + if err != nil { + return err + } + err = r.ensureNsxServiceAccount(ctx, antreaConfig, cluster) + return err +} + +func (r *AntreaConfigReconciler) deregisterAntreaNSX(ctx context.Context, antreaConfig *cniv1alpha1.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error { + if !antreaConfig.Spec.AntreaNsx.Enable { + r.Log.Info("antreaNsx is not enabled, there is no ProviderServiceAccount or NsxServiceAccount to be deleted") + return nil + } + vsphereCluster, err := cutil.VSphereClusterParavirtualForCAPICluster(ctx, r.Client, cluster) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + clusterName, exists := getClusterName(antreaConfig) + if !exists { + return fmt.Errorf("invalid antreaConfig Name") + } + account := &nsxoperatorapi.NSXServiceAccount{} + + account.Name = r.getNSXServiceAccountName(clusterName) + account.Namespace = antreaConfig.Namespace + err = r.Client.Delete(ctx, account) + if err != nil && !apierrors.IsNotFound(err) { + r.Log.Error(err, "failed to delete NSXServiceAccount", account.Namespace, account.Name) + return err + } + + provider := &vsphere.ProviderServiceAccount{} + provider.Name = r.getProviderServiceAccountName(vsphereCluster.Name) + provider.Namespace = vsphereCluster.Namespace + err = r.Client.Delete(ctx, provider) + if err != nil && !apierrors.IsNotFound(err) { + r.Log.Error(err, "failed to delete ProviderServiceAccount", provider.Namespace, provider.Name) + return err + } return nil } @@ -200,7 +408,7 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfigDataValue( antreaDataValuesSecret.StringData = make(map[string]string) // marshall the yaml contents - antreaConfigYaml, err := mapAntreaConfigSpec(cluster, antreaConfig) + antreaConfigYaml, err := mapAntreaConfigSpec(cluster, antreaConfig, r.Client) if err != nil { return err } diff --git a/addons/controllers/antrea/antreaconfig_util.go b/addons/controllers/antrea/antreaconfig_util.go index 87c4fa267f..3af9a00965 100644 --- a/addons/controllers/antrea/antreaconfig_util.go +++ b/addons/controllers/antrea/antreaconfig_util.go @@ -21,14 +21,60 @@ import ( // AntreaConfigSpec defines the desired state of AntreaConfig type antreaConfigSpec struct { - InfraProvider string `yaml:"infraProvider"` - Antrea antrea `yaml:"antrea,omitempty"` + InfraProvider string `yaml:"infraProvider"` + Antrea antrea `yaml:"antrea,omitempty"` + AntreaNsx antreaNsx `yaml:"antreaNsx,omitempty"` } type antrea struct { AntreaConfigDataValue antreaConfigDataValue `yaml:"config,omitempty"` } +type antreaNsx struct { + Enable bool `yaml:"enable,omitempty"` + BootstrapFrom AntreaNsxBootstrapFrom `yaml:"bootstrapFrom,omitempty"` + AntreaNsxConfig antreaNsxConfig `yaml:"config,omitempty"` +} + +type antreaNsxProvider struct { + ApiVersion string `yaml:"apiVersion,omitempty"` + Kind string `yaml:"kind,omitempty"` + Name string `yaml:"kind,omitempty"` +} + +type AntreaNsxBootstrapFrom struct { + // ProviderRef is used with uTKG, which will be filled by NCP operator + ProviderRef *antreaNsxProvider `yaml:"providerRef,omitempty"` + // Inline is used with TKGm, user need to fill in manually + Inline *antreaNsxInline `yaml:"inline,omitempty"` +} + +type AntreaNsxProvider struct { + // Api version for nsxServiceAccount, its value is "nsx.vmware.com/v1alpha1" now + ApiVersion string `yaml:"apiVersion,omitempty"` + // Its value is NsxServiceAccount + Kind string `yaml:"kind,omitempty"` + // Name is the name for NsxServiceAccount + Name string `yaml:"name,omitempty"` +} + +type nsxCertRef struct { + // TLSCert is cert file to access nsx manager + TLSCert string `yaml:"tls.crt,omitempty"` + // TLSKey is key file to access nsx manager + TLSKey string `yaml:"tls.key,omitempty"` +} + +type antreaNsxInline struct { + NsxManagers []string `yaml:"nsxManagers,omitempty"` + ClusterName string `yaml:"clusterName,omitempty"` + NsxCertRef nsxCertRef `yaml:"NsxCert,omitempty"` +} + +type antreaNsxConfig struct { + InfraType string `yaml:"infraType,omitempty"` +} + type antreaEgress struct { EgressExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"` } @@ -73,9 +119,9 @@ type antreaConfigDataValue struct { Multicast antreaMulticast `yaml:"multicast,omitempty"` MultiCluster antreaMultiCluster `yaml:"multicluster,omitempty"` KubeAPIServerOverride string `yaml:"kubeAPIServerOverride,omitempty"` - transportInterface string `yaml:"transportInterface,omitempty"` - transportInterfaceCIDRs []string `yaml:"transportInterfaceCIDRs,omitempty"` - multicastInterfaces []string `yaml:"multicastInterfaces,omitempty"` + TransportInterface string `yaml:"transportInterface,omitempty"` + TransportInterfaceCIDRs []string `yaml:"transportInterfaceCIDRs,omitempty"` + MulticastInterfaces []string `yaml:"multicastInterfaces,omitempty"` TunnelType string `yaml:"tunnelType,omitempty"` TrafficEncryptionMode string `yaml:"trafficEncryptionMode,omitempty"` EnableUsageReporting bool `yaml:"enableUsageReporting,omitempty"` @@ -161,7 +207,7 @@ func (r *AntreaConfigReconciler) ClusterToAntreaConfig(o client.Object) []ctrl.R return requests } -func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.AntreaConfig) (*antreaConfigSpec, error) { +func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.AntreaConfig, client client.Client) (*antreaConfigSpec, error) { configSpec := &antreaConfigSpec{} // Derive InfraProvider from the cluster @@ -197,9 +243,9 @@ func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.An configSpec.Antrea.AntreaConfigDataValue.MultiCluster.Enable = config.Spec.Antrea.AntreaConfigDataValue.MultiCluster.Enable configSpec.Antrea.AntreaConfigDataValue.MultiCluster.Namespace = config.Spec.Antrea.AntreaConfigDataValue.MultiCluster.Namespace configSpec.Antrea.AntreaConfigDataValue.KubeAPIServerOverride = config.Spec.Antrea.AntreaConfigDataValue.KubeAPIServerOverride - configSpec.Antrea.AntreaConfigDataValue.transportInterface = config.Spec.Antrea.AntreaConfigDataValue.TransportInterface - configSpec.Antrea.AntreaConfigDataValue.transportInterfaceCIDRs = config.Spec.Antrea.AntreaConfigDataValue.TransportInterfaceCIDRs - configSpec.Antrea.AntreaConfigDataValue.multicastInterfaces = config.Spec.Antrea.AntreaConfigDataValue.MulticastInterfaces + configSpec.Antrea.AntreaConfigDataValue.TransportInterface = config.Spec.Antrea.AntreaConfigDataValue.TransportInterface + configSpec.Antrea.AntreaConfigDataValue.TransportInterfaceCIDRs = config.Spec.Antrea.AntreaConfigDataValue.TransportInterfaceCIDRs + configSpec.Antrea.AntreaConfigDataValue.MulticastInterfaces = config.Spec.Antrea.AntreaConfigDataValue.MulticastInterfaces configSpec.Antrea.AntreaConfigDataValue.TunnelType = config.Spec.Antrea.AntreaConfigDataValue.TunnelType configSpec.Antrea.AntreaConfigDataValue.EnableUsageReporting = config.Spec.Antrea.AntreaConfigDataValue.EnableUsageReporting configSpec.Antrea.AntreaConfigDataValue.WireGuard.Port = config.Spec.Antrea.AntreaConfigDataValue.WireGuard.Port @@ -227,5 +273,8 @@ func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.An configSpec.Antrea.AntreaConfigDataValue.FeatureGates.MultiCluster = config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.MultiCluster configSpec.Antrea.AntreaConfigDataValue.FeatureGates.SecondaryNetwork = config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.SecondaryNetwork configSpec.Antrea.AntreaConfigDataValue.FeatureGates.TrafficControl = config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.TrafficControl + + //todo add nsx config here once antreaNsx ia packaged into antrea + return configSpec, nil } diff --git a/addons/controllers/antreaconfig_controller_test.go b/addons/controllers/antreaconfig_controller_test.go index 7d8a1a50c8..302573d6b2 100644 --- a/addons/controllers/antreaconfig_controller_test.go +++ b/addons/controllers/antreaconfig_controller_test.go @@ -13,6 +13,7 @@ import ( v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capvvmwarev1beta1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -43,8 +44,11 @@ var _ = Describe("AntreaConfig Reconciler and Webhooks", func() { const ( waitTimeout = waitTimeout //use this to change test speed when debugging antreaManifestsTestFile1 = "testdata/antrea-test-1.yaml" + antreamanifestsTestFile2 = "testdata/antrea-test-2.yaml" antreaTemplateConfigManifestsTestFile1 = "testdata/antrea-test-template-config-1.yaml" antreaTestCluster1 = "test-cluster-4" + antreaTestCluster2 = "test-cluster-5" + vsphereCluster1 = "test-cluster-5-6gvvc" ) JustBeforeEach(func() { @@ -207,6 +211,145 @@ var _ = Describe("AntreaConfig Reconciler and Webhooks", func() { }) + Context("Reconcile AntreaConfig with antreaNsx enabled for management cluster", func() { + + BeforeEach(func() { + clusterName = antreaTestCluster2 + clusterNamespace = defaultString + configCRName = util.GeneratePackageSecretName(clusterName, constants.AntreaDefaultRefName) + clusterResourceFilePath = antreamanifestsTestFile2 + }) + + It("Should reconcile AntreaConfig and create data value secret on management cluster", func() { + + clusterKey := client.ObjectKey{ + Namespace: clusterNamespace, + Name: clusterName, + } + configKey := client.ObjectKey{ + Namespace: clusterNamespace, + Name: configCRName, + } + + cluster := &clusterapiv1beta1.Cluster{} + Eventually(func() bool { + err := k8sClient.Get(ctx, clusterKey, cluster) + return err == nil + }, waitTimeout, pollingInterval).Should(BeTrue()) + + config := &cniv1alpha1.AntreaConfig{} + Eventually(func() bool { + err := k8sClient.Get(ctx, configKey, config) + if err != nil { + return false + } + + // Check owner reference + if len(config.OwnerReferences) == 0 { + return false + } + + Expect(len(config.OwnerReferences)).Should(Equal(1)) + Expect(config.OwnerReferences[0].Name).Should(Equal(clusterName)) + + Expect(config.Spec.Antrea.AntreaConfigDataValue.TrafficEncapMode).Should(Equal("encap")) + Expect(config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaTraceflow).Should(Equal(false)) + Expect(config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaPolicy).Should(Equal(true)) + Expect(config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.FlowExporter).Should(Equal(false)) + Expect(config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaIPAM).Should(Equal(false)) + Expect(config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.ServiceExternalIP).Should(Equal(false)) + Expect(config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.Multicast).Should(Equal(false)) + Expect(config.Spec.AntreaNsx.Enable).Should(Equal(true)) + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + cluster := &clusterapiv1beta1.Cluster{} + err := k8sClient.Get(ctx, clusterKey, cluster) + if err != nil { + return false + } + + serviceCIDR, serviceCIDRv6, err := util.GetServiceCIDRs(cluster) + if err != nil { + return false + } + + infraProvider, err := util.GetInfraProvider(cluster) + if err != nil { + return false + } + + // Check infraProvider values + Expect(infraProvider).Should(Equal("vsphere")) + + // Check ServiceCIDR and ServiceCIDRv6 values + Expect(serviceCIDR).Should(Equal("192.168.0.0/16")) + Expect(serviceCIDRv6).Should(Equal("fd00:100:96::/48")) + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + secretKey := client.ObjectKey{ + Namespace: clusterNamespace, + Name: util.GenerateDataValueSecretName(clusterName, constants.AntreaAddonName), + } + secret := &v1.Secret{} + err := k8sClient.Get(ctx, secretKey, secret) + if err != nil { + return false + } + Expect(secret.Type).Should(Equal(v1.SecretTypeOpaque)) + + // check data value secret contents + secretData := string(secret.Data["values.yaml"]) + + Expect(strings.Contains(secretData, "serviceCIDR: 192.168.0.0/16")).Should(BeTrue()) + Expect(strings.Contains(secretData, "serviceCIDRv6: fd00:100:96::/48")).Should(BeTrue()) + Expect(strings.Contains(secretData, "infraProvider: vsphere")).Should(BeTrue()) + + Expect(strings.Contains(secretData, "trafficEncapMode: encap")).Should(BeTrue()) + Expect(strings.Contains(secretData, "tlsCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384")).Should(BeTrue()) + Expect(strings.Contains(secretData, "AntreaProxy: true")).Should(BeTrue()) + Expect(strings.Contains(secretData, "AntreaPolicy: true")).Should(BeTrue()) + + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + // Check status.secretRef after reconciliation + config := &cniv1alpha1.AntreaConfig{} + err := k8sClient.Get(ctx, configKey, config) + if err != nil { + return false + } + return config.Status.SecretRef == util.GenerateDataValueSecretName(clusterName, constants.AntreaAddonName) + }, waitTimeout, pollingInterval).Should(BeTrue()) + + // check if ProviderServiceAccount is created + By("check ProviderServiceAccount is created") + serviceAccount := &capvvmwarev1beta1.ProviderServiceAccount{} + Eventually(func() bool { + serviceAccountKey := client.ObjectKey{ + Namespace: defaultString, + Name: fmt.Sprintf("%s-antrea", vsphereCluster1), + } + if err := k8sClient.Get(ctx, serviceAccountKey, serviceAccount); err != nil { + return false + } + Expect(serviceAccount.Spec.Ref.Name).To(Equal(vsphereCluster1)) + Expect(serviceAccount.Spec.Ref.Namespace).To(Equal(serviceAccountKey.Namespace)) + Expect(serviceAccount.Spec.Rules).To(HaveLen(2)) + Expect(serviceAccount.Spec.TargetNamespace).To(Equal("vmware-system-antrea")) + Expect(serviceAccount.Spec.TargetSecretName).To(Equal("supervisor-cred")) + return true + }) + + // TODO: we shall check if nsxServiceAccount is created, but since it not registered, we do not check now + }) + + }) + Context("Reconcile AntreaConfig used as template", func() { BeforeEach(func() { diff --git a/addons/controllers/testdata/antrea-test-2.yaml b/addons/controllers/testdata/antrea-test-2.yaml new file mode 100644 index 0000000000..a19bc7afde --- /dev/null +++ b/addons/controllers/testdata/antrea-test-2.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test-cluster-5 + namespace: default +spec: + infrastructureRef: + kind: VSphereCluster + clusterNetwork: + pods: + cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ] + services: + cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ] + topology: + class: test-clusterclass-tcbt + version: v1.22.2 +--- +apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereCluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: test-cluster-5 + topology.cluster.x-k8s.io/owned: "" + name: test-cluster-5-6gvvc + namespace: default +spec: + controlPlaneEndpoint: + host: 192.168.123.3 + port: 6443 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: test-cluster-5 +--- +apiVersion: cni.tanzu.vmware.com/v1alpha1 +kind: AntreaConfig +metadata: + name: test-cluster-5-antrea-package + namespace: default + ownerReferences: + - apiVersion: cluster.x-k8s.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: Cluster + name: test-cluster-5 + uid: cbd29b10-c190-422e-86f1-a0321d1aab7d +spec: + antrea: + config: + egress: + exceptCIDRs: [] + nodePortLocal: + enabled: true + portRange: 61000-62000 + antreaProxy: + proxyAll: false + nodePortAddresses: [] + skipServices: [] + proxyLoadBalancerIPs: false + flowExporter: + collectorAddress: "flow-aggregator.flow-aggregator.svc:4739:tls" + pollInterval: "5s" + activeFlowTimeout: "30s" + idleFlowTimeout: "15s" + kubeAPIServerOverride: null + transportInterface: null + transportInterfaceCIDRs: [] + multicastInterfaces: [] + tunnelType: geneve + trafficEncryptionMode: none + wireGuard: + port: 51820 + serviceCIDR: 10.96.0.0/12 + serviceCIDRv6: null + enableUsageReporting: false + trafficEncapMode: encap + noSNAT: false + disableUdpTunnelOffload: false + defaultMTU: "" + tlsCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384 + featureGates: + AntreaProxy: true + EndpointSlice: false + AntreaPolicy: true + FlowExporter: false + Egress: false + NodePortLocal: true + AntreaTraceflow: false + NetworkPolicyStats: false + AntreaIPAM: false + ServiceExternalIP: false + Multicast: false + antreaNsx: + enable: true diff --git a/addons/go.mod b/addons/go.mod index 08bd958031..8b2c048d70 100644 --- a/addons/go.mod +++ b/addons/go.mod @@ -18,6 +18,7 @@ require ( github.com/vmware-tanzu/carvel-kapp-controller v0.35.0 github.com/vmware-tanzu/carvel-secretgen-controller v0.5.0 github.com/vmware-tanzu/carvel-vendir v0.26.0 + github.com/vmware-tanzu/nsx-operator v0.0.0-20221102072746-f7b4cc7cce41 github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs v0.0.0-20220907220230-c1137d344dd3 github.com/vmware-tanzu/tanzu-framework/apis/run v0.0.0-20220907220230-c1137d344dd3 github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20211202185235-43eb44c09ecd @@ -96,7 +97,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/vmware-tanzu/net-operator-api v0.0.0-20210401185409-b0dc6c297707 // indirect github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20211209213435-0f4ab286f64f // indirect - github.com/vmware/govmomi v0.27.1 // indirect + github.com/vmware/govmomi v0.27.4 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.22.0 // indirect diff --git a/addons/go.sum b/addons/go.sum index d9b222d191..8c5c5caf95 100644 --- a/addons/go.sum +++ b/addons/go.sum @@ -882,14 +882,16 @@ github.com/vmware-tanzu/carvel-vendir v0.26.0 h1:Q98tPnH9WUAWE2vJSAP0lsHGsfwGjfW github.com/vmware-tanzu/carvel-vendir v0.26.0/go.mod h1:JcuNNVONFbZTbm/GjtGiWUfFrt17YBQzeGT9+gY1+yY= github.com/vmware-tanzu/net-operator-api v0.0.0-20210401185409-b0dc6c297707 h1:2onys8tWlQh7DFiOz6+68AwJdW9EBOEv6RTKzwh1x7A= github.com/vmware-tanzu/net-operator-api v0.0.0-20210401185409-b0dc6c297707/go.mod h1:pDB0pUiFYufuP3lUkQX9fZ67PYnKvqBpDcJN3mSrw5U= +github.com/vmware-tanzu/nsx-operator v0.0.0-20221102072746-f7b4cc7cce41 h1:GM/KH7xmJQ8uPzrCQf3B1dhVbpIwYNPAmxLeWdKtcEA= +github.com/vmware-tanzu/nsx-operator v0.0.0-20221102072746-f7b4cc7cce41/go.mod h1:Exsg90TOJ0FQBFzmdYurJNVCh9PMN3IkAr/tA+rKrLQ= github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20211202185235-43eb44c09ecd h1:BXz4aAPzRAYD8x8LEhjEsmvTj9mCvesnr4ApT1Ay4YY= github.com/vmware-tanzu/vm-operator-api v0.1.4-0.20211202185235-43eb44c09ecd/go.mod h1:mubK0QMyaA2TbeAmGsu2GVfiqDFppNUAUqoMPoKFgzM= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20211209213435-0f4ab286f64f h1:RUuS5lh25citvQoXmDSfxJ1BB72LXOjD5cXvJETJ7Cc= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20211209213435-0f4ab286f64f/go.mod h1:5rqRJ9zGR+KnKbkGx373WgN8xJpvAj99kHnfoDYRO5I= github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20211209213435-0f4ab286f64f h1:wwYUf16/g8bLywQMQJB5VHbDtuf6aOFH24Ar2/yA7+I= github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20211209213435-0f4ab286f64f/go.mod h1:dfYrWS8DMRN+XZfhu8M4LVHmeGvYB29Ipd7j4uIq+mU= -github.com/vmware/govmomi v0.27.1 h1:Rf3o1btFrkJa9be5KtgJ4CyOO8mbFnBxmNtAVHNyFes= -github.com/vmware/govmomi v0.27.1/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o= +github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I= +github.com/vmware/govmomi v0.27.4/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/addons/main.go b/addons/main.go index bc7bc10cb3..ec838833d4 100644 --- a/addons/main.go +++ b/addons/main.go @@ -33,6 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" + nsxoperator "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + kappctrl "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" kapppkg "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" kappdatapkg "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" @@ -84,6 +86,7 @@ func init() { _ = capvvmwarev1beta1.AddToScheme(scheme) _ = vmoperatorv1alpha1.AddToScheme(scheme) _ = topologyv1alpha1.AddToScheme(scheme) + _ = nsxoperator.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } diff --git a/addons/pkg/constants/constants.go b/addons/pkg/constants/constants.go index ac640c288b..d7e547fa12 100644 --- a/addons/pkg/constants/constants.go +++ b/addons/pkg/constants/constants.go @@ -282,6 +282,9 @@ const ( // logic AddCBMissingFieldsAnnotationKey = "tkg.tanzu.vmware.com/add-missing-fields-from-tkr" + // VsphereAntreaConfigProviderServiceAccountAggregatedClusterRole is the name of ClusterRole created by controllers that use ProviderServiceAccount + VsphereAntreaConfigProviderServiceAccountAggregatedClusterRole = "addons-vsphere-antreaConfig-providerserviceaccount-aggregatedrole" + // VsphereCPIProviderServiceAccountAggregatedClusterRole is the name of ClusterRole created by controllers that use ProviderServiceAccount VsphereCPIProviderServiceAccountAggregatedClusterRole = "addons-vsphere-cpi-providerserviceaccount-aggregatedrole" diff --git a/apis/addonconfigs/cni/v1alpha1/antreaconfig_types.go b/apis/addonconfigs/cni/v1alpha1/antreaconfig_types.go index 70663f9f62..87f390be79 100644 --- a/apis/addonconfigs/cni/v1alpha1/antreaconfig_types.go +++ b/apis/addonconfigs/cni/v1alpha1/antreaconfig_types.go @@ -12,6 +12,8 @@ type AntreaProxyNodePortAddress []string // AntreaConfigSpec defines the desired state of AntreaConfig type AntreaConfigSpec struct { Antrea Antrea `json:"antrea,omitempty"` + // AntreaNsx defines nsxt adapter related configurations + AntreaNsx AntreaNsx `json:"antreaNsx,omitempty"` } type Antrea struct { @@ -258,11 +260,66 @@ type AntreaFeatureGates struct { // AntreaConfigStatus defines the observed state of AntreaConfig type AntreaConfigStatus struct { + // Message to indicate failure reason + // +kubebuilder:validation:Optional + Message string `json:"message,omitempty"` // Reference to the data value secret created by controller // +kubebuilder:validation:Optional SecretRef string `json:"secretRef,omitempty"` } +type AntreaNsx struct { + // Enable indicates whether nsxt adapter shall be enabled in the cluster + // +kubebuilder:validation:Optional + // +kubebuilder:default:=false + Enable bool `json:"enable,omitempty"` + // BootstrapFrom either providerRef or inline configs + // +kubebuilder:validation:Optional + BootstrapFrom AntreaNsxBootstrapFrom `json:"bootstrapFrom,omitempty"` + // Config is configuration for nsxt adapter + // +kubebuilder:validation:Optional + AntreaNsxConfig AntreaNsxConfig `json:"config,omitempty"` +} + +type AntreaNsxBootstrapFrom struct { + // ProviderRef is used with uTKG, which will be filled by uTKG Addon Controller + // +kubebuilder:validation:Optional + ProviderRef *AntreaNsxProvider `json:"providerRef,omitempty"` + // Inline is used with TKGm, user need to fill in manually + // +kubebuilder:validation:Optional + Inline *AntreaNsxInline `json:"inline,omitempty"` +} + +type AntreaNsxProvider struct { + // Api version for nsxServiceAccount, its value is "nsx.vmware.com/v1alpha1" now + // +kubebuilder:validation:Optional + ApiGroup string `json:"apigroup,omitempty"` + // Kind is the kind for crd, here its value is NsxServiceAccount + // +kubebuilder:validation:Optional + Kind string `json:"kind,omitempty"` + // Name is the name for NsxServiceAccount + // +kubebuilder:validation:Optional + Name string `json:"name,omitempty"` +} + +type AntreaNsxInline struct { + // NsxManagers is the list for nsx managers, it can be either IP address or domain name + // +kubebuilder:validation:Optional + NsxManagers []string `json:"nsxManagers,omitempty"` + // ClusterName is the name for the created cluster + // +kubebuilder:validation:Optional + ClusterName string `json:"clusterName,omitempty"` + // NsxCertName is cert files to access nsx manager + // +kubebuilder:validation:Optional + NsxCertName string `json:"nsxCertName,omitempty"` +} + +type AntreaNsxConfig struct { + // InfraType is the type for infrastructure, so far it is vSphere, VMC, AWS, Azure + // +kubebuilder:validation:Optional + InfraType string `json:"infraType,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=antreaconfigs,shortName=antreaconf,scope=Namespaced diff --git a/apis/addonconfigs/cni/v1alpha1/zz_generated.deepcopy.go b/apis/addonconfigs/cni/v1alpha1/zz_generated.deepcopy.go index 743e21d154..4873e67a0a 100644 --- a/apis/addonconfigs/cni/v1alpha1/zz_generated.deepcopy.go +++ b/apis/addonconfigs/cni/v1alpha1/zz_generated.deepcopy.go @@ -124,6 +124,7 @@ func (in *AntreaConfigList) DeepCopyObject() runtime.Object { func (in *AntreaConfigSpec) DeepCopyInto(out *AntreaConfigSpec) { *out = *in in.Antrea.DeepCopyInto(&out.Antrea) + in.AntreaNsx.DeepCopyInto(&out.AntreaNsx) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AntreaConfigSpec. @@ -246,6 +247,98 @@ func (in *AntreaNodePortLocal) DeepCopy() *AntreaNodePortLocal { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AntreaNsx) DeepCopyInto(out *AntreaNsx) { + *out = *in + in.BootstrapFrom.DeepCopyInto(&out.BootstrapFrom) + out.AntreaNsxConfig = in.AntreaNsxConfig +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AntreaNsx. +func (in *AntreaNsx) DeepCopy() *AntreaNsx { + if in == nil { + return nil + } + out := new(AntreaNsx) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AntreaNsxBootstrapFrom) DeepCopyInto(out *AntreaNsxBootstrapFrom) { + *out = *in + if in.ProviderRef != nil { + in, out := &in.ProviderRef, &out.ProviderRef + *out = new(AntreaNsxProvider) + **out = **in + } + if in.Inline != nil { + in, out := &in.Inline, &out.Inline + *out = new(AntreaNsxInline) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AntreaNsxBootstrapFrom. +func (in *AntreaNsxBootstrapFrom) DeepCopy() *AntreaNsxBootstrapFrom { + if in == nil { + return nil + } + out := new(AntreaNsxBootstrapFrom) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AntreaNsxConfig) DeepCopyInto(out *AntreaNsxConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AntreaNsxConfig. +func (in *AntreaNsxConfig) DeepCopy() *AntreaNsxConfig { + if in == nil { + return nil + } + out := new(AntreaNsxConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AntreaNsxInline) DeepCopyInto(out *AntreaNsxInline) { + *out = *in + if in.NsxManagers != nil { + in, out := &in.NsxManagers, &out.NsxManagers + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AntreaNsxInline. +func (in *AntreaNsxInline) DeepCopy() *AntreaNsxInline { + if in == nil { + return nil + } + out := new(AntreaNsxInline) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AntreaNsxProvider) DeepCopyInto(out *AntreaNsxProvider) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AntreaNsxProvider. +func (in *AntreaNsxProvider) DeepCopy() *AntreaNsxProvider { + if in == nil { + return nil + } + out := new(AntreaNsxProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AntreaProxy) DeepCopyInto(out *AntreaProxy) { *out = *in diff --git a/apis/addonconfigs/config/crd/bases/cni.tanzu.vmware.com_antreaconfigs.yaml b/apis/addonconfigs/config/crd/bases/cni.tanzu.vmware.com_antreaconfigs.yaml index 1eae2cafca..808e3ab345 100644 --- a/apis/addonconfigs/config/crd/bases/cni.tanzu.vmware.com_antreaconfigs.yaml +++ b/apis/addonconfigs/config/crd/bases/cni.tanzu.vmware.com_antreaconfigs.yaml @@ -280,10 +280,67 @@ spec: type: object type: object type: object + antreaNsx: + description: AntreaNsx defines nsxt adapter related configurations + properties: + bootstrapFrom: + description: BootstrapFrom either providerRef or inline configs + properties: + inline: + description: Inline is used with TKGm, user need to fill in + manually + properties: + clusterName: + description: ClusterName is the name for the created cluster + type: string + nsxCertName: + description: NsxCertName is cert files to access nsx manager + type: string + nsxManagers: + description: NsxManagers is the list for nsx managers, + it can be either IP address or domain name + items: + type: string + type: array + type: object + providerRef: + description: ProviderRef is used with uTKG, which will be + filled by uTKG Addon Controller + properties: + apigroup: + description: Api version for nsxServiceAccount, its value + is "nsx.vmware.com/v1alpha1" now + type: string + kind: + description: Kind is the kind for crd, here its value + is NsxServiceAccount + type: string + name: + description: Name is the name for NsxServiceAccount + type: string + type: object + type: object + config: + description: Config is configuration for nsxt adapter + properties: + infraType: + description: InfraType is the type for infrastructure, so + far it is vSphere, VMC, AWS, Azure + type: string + type: object + enable: + default: false + description: Enable indicates whether nsxt adapter shall be enabled + in the cluster + type: boolean + type: object type: object status: description: AntreaConfigStatus defines the observed state of AntreaConfig properties: + message: + description: Message to indicate failure reason + type: string secretRef: description: Reference to the data value secret created by controller type: string diff --git a/packages/addons-manager/bundle/config/upstream/addonconfigscrds/cni.tanzu.vmware.com_antreaconfigs.yaml b/packages/addons-manager/bundle/config/upstream/addonconfigscrds/cni.tanzu.vmware.com_antreaconfigs.yaml index 1eae2cafca..808e3ab345 100644 --- a/packages/addons-manager/bundle/config/upstream/addonconfigscrds/cni.tanzu.vmware.com_antreaconfigs.yaml +++ b/packages/addons-manager/bundle/config/upstream/addonconfigscrds/cni.tanzu.vmware.com_antreaconfigs.yaml @@ -280,10 +280,67 @@ spec: type: object type: object type: object + antreaNsx: + description: AntreaNsx defines nsxt adapter related configurations + properties: + bootstrapFrom: + description: BootstrapFrom either providerRef or inline configs + properties: + inline: + description: Inline is used with TKGm, user need to fill in + manually + properties: + clusterName: + description: ClusterName is the name for the created cluster + type: string + nsxCertName: + description: NsxCertName is cert files to access nsx manager + type: string + nsxManagers: + description: NsxManagers is the list for nsx managers, + it can be either IP address or domain name + items: + type: string + type: array + type: object + providerRef: + description: ProviderRef is used with uTKG, which will be + filled by uTKG Addon Controller + properties: + apigroup: + description: Api version for nsxServiceAccount, its value + is "nsx.vmware.com/v1alpha1" now + type: string + kind: + description: Kind is the kind for crd, here its value + is NsxServiceAccount + type: string + name: + description: Name is the name for NsxServiceAccount + type: string + type: object + type: object + config: + description: Config is configuration for nsxt adapter + properties: + infraType: + description: InfraType is the type for infrastructure, so + far it is vSphere, VMC, AWS, Azure + type: string + type: object + enable: + default: false + description: Enable indicates whether nsxt adapter shall be enabled + in the cluster + type: boolean + type: object type: object status: description: AntreaConfigStatus defines the observed state of AntreaConfig properties: + message: + description: Message to indicate failure reason + type: string secretRef: description: Reference to the data value secret created by controller type: string From 4713aedd9ae29e9ccd2dee91c01882a010eeb1e8 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Mon, 5 Dec 2022 13:55:32 +0800 Subject: [PATCH 2/2] make antreaConfig structs public Signed-off-by: Bin Liu --- .../controllers/antrea/antreaconfig_util.go | 18 +++---- .../antreaconfig_controller_test.go | 49 +++++++++++-------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/addons/controllers/antrea/antreaconfig_util.go b/addons/controllers/antrea/antreaconfig_util.go index 3af9a00965..06618bc704 100644 --- a/addons/controllers/antrea/antreaconfig_util.go +++ b/addons/controllers/antrea/antreaconfig_util.go @@ -20,7 +20,7 @@ import ( ) // AntreaConfigSpec defines the desired state of AntreaConfig -type antreaConfigSpec struct { +type AntreaConfigSpec struct { InfraProvider string `yaml:"infraProvider"` Antrea antrea `yaml:"antrea,omitempty"` AntreaNsx antreaNsx `yaml:"antreaNsx,omitempty"` @@ -32,24 +32,18 @@ type antrea struct { type antreaNsx struct { Enable bool `yaml:"enable,omitempty"` - BootstrapFrom AntreaNsxBootstrapFrom `yaml:"bootstrapFrom,omitempty"` + BootstrapFrom antreaNsxBootstrapFrom `yaml:"bootstrapFrom,omitempty"` AntreaNsxConfig antreaNsxConfig `yaml:"config,omitempty"` } -type antreaNsxProvider struct { - ApiVersion string `yaml:"apiVersion,omitempty"` - Kind string `yaml:"kind,omitempty"` - Name string `yaml:"kind,omitempty"` -} - -type AntreaNsxBootstrapFrom struct { +type antreaNsxBootstrapFrom struct { // ProviderRef is used with uTKG, which will be filled by NCP operator ProviderRef *antreaNsxProvider `yaml:"providerRef,omitempty"` // Inline is used with TKGm, user need to fill in manually Inline *antreaNsxInline `yaml:"inline,omitempty"` } -type AntreaNsxProvider struct { +type antreaNsxProvider struct { // Api version for nsxServiceAccount, its value is "nsx.vmware.com/v1alpha1" now ApiVersion string `yaml:"apiVersion,omitempty"` // Its value is NsxServiceAccount @@ -207,8 +201,8 @@ func (r *AntreaConfigReconciler) ClusterToAntreaConfig(o client.Object) []ctrl.R return requests } -func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.AntreaConfig, client client.Client) (*antreaConfigSpec, error) { - configSpec := &antreaConfigSpec{} +func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha1.AntreaConfig, client client.Client) (*AntreaConfigSpec, error) { + configSpec := &AntreaConfigSpec{} // Derive InfraProvider from the cluster infraProvider, err := util.GetInfraProvider(cluster) diff --git a/addons/controllers/antreaconfig_controller_test.go b/addons/controllers/antreaconfig_controller_test.go index 302573d6b2..ba571bf929 100644 --- a/addons/controllers/antreaconfig_controller_test.go +++ b/addons/controllers/antreaconfig_controller_test.go @@ -6,10 +6,10 @@ package controllers import ( "fmt" "os" - "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,6 +17,7 @@ import ( clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" + antreatype "github.com/vmware-tanzu/tanzu-framework/addons/controllers/antrea" cutil "github.com/vmware-tanzu/tanzu-framework/addons/controllers/utils" "github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants" "github.com/vmware-tanzu/tanzu-framework/addons/pkg/util" @@ -183,16 +184,19 @@ var _ = Describe("AntreaConfig Reconciler and Webhooks", func() { Expect(secret.Type).Should(Equal(v1.SecretTypeOpaque)) // check data value secret contents - secretData := string(secret.Data["values.yaml"]) - - Expect(strings.Contains(secretData, "serviceCIDR: 192.168.0.0/16")).Should(BeTrue()) - Expect(strings.Contains(secretData, "serviceCIDRv6: fd00:100:96::/48")).Should(BeTrue()) - Expect(strings.Contains(secretData, "infraProvider: docker")).Should(BeTrue()) - - Expect(strings.Contains(secretData, "trafficEncapMode: encap")).Should(BeTrue()) - Expect(strings.Contains(secretData, "tlsCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384")).Should(BeTrue()) - Expect(strings.Contains(secretData, "AntreaProxy: true")).Should(BeTrue()) - Expect(strings.Contains(secretData, "AntreaPolicy: true")).Should(BeTrue()) + content := secret.Data["values.yaml"] + spec := antreatype.AntreaConfigSpec{} + err = yaml.Unmarshal(content, &spec) + if err != nil { + return false + } + Expect(spec.Antrea.AntreaConfigDataValue.ServiceCIDR).Should(Equal("192.168.0.0/16")) + Expect(spec.Antrea.AntreaConfigDataValue.ServiceCIDRv6).Should(Equal("fd00:100:96::/48")) + Expect(spec.InfraProvider).Should(Equal("docker")) + Expect(spec.Antrea.AntreaConfigDataValue.TrafficEncapMode).Should(Equal("encap")) + Expect(spec.Antrea.AntreaConfigDataValue.TLSCipherSuites).Should(Equal("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384")) + Expect(spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaProxy).Should(Equal(true)) + Expect(spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaPolicy).Should(Equal(true)) return true }, waitTimeout, pollingInterval).Should(BeTrue()) @@ -302,16 +306,19 @@ var _ = Describe("AntreaConfig Reconciler and Webhooks", func() { Expect(secret.Type).Should(Equal(v1.SecretTypeOpaque)) // check data value secret contents - secretData := string(secret.Data["values.yaml"]) - - Expect(strings.Contains(secretData, "serviceCIDR: 192.168.0.0/16")).Should(BeTrue()) - Expect(strings.Contains(secretData, "serviceCIDRv6: fd00:100:96::/48")).Should(BeTrue()) - Expect(strings.Contains(secretData, "infraProvider: vsphere")).Should(BeTrue()) - - Expect(strings.Contains(secretData, "trafficEncapMode: encap")).Should(BeTrue()) - Expect(strings.Contains(secretData, "tlsCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384")).Should(BeTrue()) - Expect(strings.Contains(secretData, "AntreaProxy: true")).Should(BeTrue()) - Expect(strings.Contains(secretData, "AntreaPolicy: true")).Should(BeTrue()) + content := secret.Data["values.yaml"] + spec := antreatype.AntreaConfigSpec{} + err = yaml.Unmarshal(content, &spec) + if err != nil { + return false + } + Expect(spec.Antrea.AntreaConfigDataValue.ServiceCIDR).Should(Equal("192.168.0.0/16")) + Expect(spec.Antrea.AntreaConfigDataValue.ServiceCIDRv6).Should(Equal("fd00:100:96::/48")) + Expect(spec.InfraProvider).Should(Equal("vsphere")) + Expect(spec.Antrea.AntreaConfigDataValue.TrafficEncapMode).Should(Equal("encap")) + Expect(spec.Antrea.AntreaConfigDataValue.TLSCipherSuites).Should(Equal("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384")) + Expect(spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaProxy).Should(Equal(true)) + Expect(spec.Antrea.AntreaConfigDataValue.FeatureGates.AntreaPolicy).Should(Equal(true)) return true }, waitTimeout, pollingInterval).Should(BeTrue())