diff --git a/Makefile b/Makefile index 4fc3dd1c..6d4870df 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ e2e: kind-clusters ## Runs end-to-end tests against KinD clusters $(foreach suite, $(TEST_SUITES), \ PATH=$(LOCALBIN):$$PATH \ TAG=$$local_tag \ - go test -tags=integ -run TestTraffic $(PROJECT_DIR)/test/e2e/scenarios/$(suite) \ + go test -tags=integ -timeout 30m -run TestTraffic $(PROJECT_DIR)/test/e2e/scenarios/$(suite) \ --istio.test.hub=docker.io/istio\ --istio.test.tag=$(ISTIO_VERSION)\ --istio.test.kube.config=$(PROJECT_DIR)/test/east.kubeconfig,$(PROJECT_DIR)/test/west.kubeconfig,$(PROJECT_DIR)/test/central.kubeconfig\ diff --git a/api/v1alpha1/meshfederation_types.go b/api/v1alpha1/meshfederation_types.go index c5710add..7112fc91 100644 --- a/api/v1alpha1/meshfederation_types.go +++ b/api/v1alpha1/meshfederation_types.go @@ -48,20 +48,9 @@ type MeshFederationList struct { // MeshFederationSpec defines the desired state of MeshFederation. type MeshFederationSpec struct { - // Network name used by Istio for load balancing - // +kubebuilder:validation:Required - Network string `json:"network"` - - // +kubebuilder:default:=cluster.local - TrustDomain string `json:"trustDomain"` - - // Namespace used to create mesh-wide resources - // +kubebuilder:default:=istio-system - ControlPlaneNamespace string `json:"controlPlaneNamespace"` - // TODO: CRD proposal states "If no ingress is specified, it means the controller supports only single network topology". However, some config, such as gateway/port config, seems to be required. // Config specifying ingress type and ingress gateway config - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional IngressConfig IngressConfig `json:"ingress"` // Selects the K8s Services to export to all remote meshes. diff --git a/chart/crds/federation.openshift-service-mesh.io_meshfederations.yaml b/chart/crds/federation.openshift-service-mesh.io_meshfederations.yaml index 811660e4..8b24f9ec 100644 --- a/chart/crds/federation.openshift-service-mesh.io_meshfederations.yaml +++ b/chart/crds/federation.openshift-service-mesh.io_meshfederations.yaml @@ -39,10 +39,6 @@ spec: spec: description: MeshFederationSpec defines the desired state of MeshFederation. properties: - controlPlaneNamespace: - default: istio-system - description: Namespace used to create mesh-wide resources - type: string export: description: |- Selects the K8s Services to export to all remote meshes. @@ -154,17 +150,6 @@ spec: - gateway - type type: object - network: - description: Network name used by Istio for load balancing - type: string - trustDomain: - default: cluster.local - type: string - required: - - controlPlaneNamespace - - ingress - - network - - trustDomain type: object status: description: MeshFederationStatus defines the observed state of MeshFederation. diff --git a/chart/templates/clusterrole.yaml b/chart/templates/clusterrole.yaml index 66812402..2bbfb962 100644 --- a/chart/templates/clusterrole.yaml +++ b/chart/templates/clusterrole.yaml @@ -8,26 +8,27 @@ rules: verbs: ["get", "watch", "list"] - apiGroups: ["networking.istio.io"] resources: ["gateways", "serviceentries", "workloadentries"] - verbs: ["get", "list", "create", "update", "patch", "delete"] + verbs: ["get", "list", "create", "update", "patch", "delete", "watch"] - apiGroups: ["security.istio.io"] resources: ["peerauthentications"] - verbs: ["get", "list", "create", "update", "patch", "delete"] + verbs: ["get", "list", "create", "update", "patch", "delete", "watch"] {{- if (include "remotes.hasOpenshiftRouterPeer" .) }} - apiGroups: ["networking.istio.io"] resources: ["destinationrules"] verbs: ["get", "list", "create", "update", "patch", "delete"] {{- end }} -{{- if eq .Values.federation.meshPeers.local.ingressType "openshift-router" }} - apiGroups: ["networking.istio.io"] resources: ["envoyfilters"] - verbs: ["get", "list", "create", "update", "patch", "delete"] + verbs: ["get", "list", "create", "update", "patch", "delete", "watch"] - apiGroups: ["route.openshift.io"] resources: ["routes", "routes/custom-host"] - verbs: ["get", "list", "create", "update", "patch", "delete"] -{{- end }} + verbs: ["get", "list", "create", "update", "patch", "delete", "watch"] - apiGroups: ["federation.openshift-service-mesh.io"] resources: ["meshfederations", "federatedservices"] verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] - apiGroups: ["federation.openshift-service-mesh.io"] resources: ["meshfederations/status", "federatedservices/status"] - verbs: ["get"] + verbs: ["get", "update", "patch"] +- apiGroups: ["federation.openshift-service-mesh.io"] + resources: ["meshfederations/finalizers"] + verbs: ["get", "update", "patch"] diff --git a/cmd/federation-controller/main.go b/cmd/federation-controller/main.go index bc79de6e..679062e0 100644 --- a/cmd/federation-controller/main.go +++ b/cmd/federation-controller/main.go @@ -20,21 +20,20 @@ import ( "fmt" "os" "os/signal" - "sort" "syscall" "time" - routev1client "github.com/openshift/client-go/route/clientset/versioned" + // +kubebuilder:scaffold:imports + routev1 "github.com/openshift/api/route/v1" + networkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3" + securityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" istiokube "istio.io/istio/pkg/kube" istiolog "istio.io/istio/pkg/log" - "istio.io/istio/pkg/slices" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" clientgoscheme "k8s.io/client-go/kubernetes/scheme" v1 "k8s.io/client-go/listers/core/v1" - // +kubebuilder:scaffold:imports "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -47,13 +46,10 @@ import ( "github.com/openshift-service-mesh/federation/internal/pkg/config" "github.com/openshift-service-mesh/federation/internal/pkg/istio" "github.com/openshift-service-mesh/federation/internal/pkg/legacy/fds" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/informer" "github.com/openshift-service-mesh/federation/internal/pkg/legacy/kube" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds/adsc" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds/adss" "github.com/openshift-service-mesh/federation/internal/pkg/networking" - "github.com/openshift-service-mesh/federation/internal/pkg/openshift" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds/adsc" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -62,14 +58,9 @@ import ( var ( // Global variables to store the parsed commandline arguments - meshPeers, - exportedServiceSet, - importedServiceSet, - metricsAddr, - probeAddr string - - enableLeaderElection, - useCtrls bool + meshPeers, exportedServiceSet, importedServiceSet, + metricsAddr, probeAddr string + enableLeaderElection bool loggingOptions = istiolog.DefaultOptions() log = istiolog.RegisterScope("default", "default logging scope") @@ -80,7 +71,9 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(v1alpha1.AddToScheme(scheme)) - utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(networkingv1alpha3.AddToScheme(scheme)) + utilruntime.Must(securityv1beta1.AddToScheme(scheme)) + utilruntime.Must(routev1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -100,9 +93,6 @@ func parseFlags() { "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&useCtrls, "use-ctrls", false, - "feature-flag: enables controller-runtime reconcilers instead of legacy mode.") - // Attach Istio logging options to the flag set loggingOptions.AttachFlags(func(_ *[]string, _ string, _ []string, _ string) { // unused and not available out-of-the box in flag package @@ -115,6 +105,8 @@ func parseFlags() { } func main() { + opts := zap.Options{Development: true} + opts.BindFlags(flag.CommandLine) parseFlags() if err := istiolog.Configure(loggingOptions); err != nil { @@ -126,21 +118,6 @@ func main() { log.Fatalf("failed to parse configuration passed to the program arguments: %v", err) } - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer cancel() - - if useCtrls { - runCtrls(ctx, cancel) - } - - runLegacyMode(ctx, cfg) - - <-ctx.Done() -} - -func runCtrls(ctx context.Context, cancel context.CancelFunc) { - opts := zap.Options{Development: true} - opts.BindFlags(flag.CommandLine) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ @@ -157,7 +134,9 @@ func runCtrls(ctx context.Context, cancel context.CancelFunc) { os.Exit(1) } - if err = meshfederation.NewReconciler(mgr.GetClient()).SetupWithManager(mgr); err != nil { + meshConfigPushRequests := make(chan xds.PushRequest) + + if err = meshfederation.NewReconciler(mgr.GetClient(), cfg.MeshPeers.Remotes, meshConfigPushRequests).SetupWithManager(mgr); err != nil { log.Errorf("unable to create controller for MeshFederation custom resource: %s", err) os.Exit(1) } @@ -166,6 +145,7 @@ func runCtrls(ctx context.Context, cancel context.CancelFunc) { os.Exit(1) } // +kubebuilder:scaffold:builder + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { log.Errorf("unable to set up health check: %s", err) os.Exit(1) @@ -174,6 +154,10 @@ func runCtrls(ctx context.Context, cancel context.CancelFunc) { log.Errorf("unable to set up ready check: %s", err) os.Exit(1) } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + go func() { log.Info("starting manager") if err := mgr.Start(ctx); err != nil { @@ -181,9 +165,7 @@ func runCtrls(ctx context.Context, cancel context.CancelFunc) { cancel() } }() -} -func runLegacyMode(ctx context.Context, cfg *config.Federation) { kubeConfig, err := rest.InClusterConfig() if err != nil { log.Fatalf("failed to create in-cluster config: %v", err) @@ -194,33 +176,17 @@ func runLegacyMode(ctx context.Context, cfg *config.Federation) { log.Fatalf("failed to create Istio client: %v", err) } - fdsPushRequests := make(chan xds.PushRequest) - meshConfigPushRequests := make(chan xds.PushRequest) - - informerFactory := informers.NewSharedInformerFactory(istioClient.Kube(), 0) - serviceInformer := informerFactory.Core().V1().Services().Informer() - serviceLister := informerFactory.Core().V1().Services().Lister() - informerFactory.Start(ctx.Done()) - - serviceController, err := informer.NewResourceController(serviceInformer, corev1.Service{}, - informer.NewServiceExportEventHandler(*cfg, fdsPushRequests, meshConfigPushRequests)) - if err != nil { - log.Fatalf("failed to create service informer: %v", err) - } - serviceController.RunAndWait(ctx.Done()) - - startFederationServer(ctx, cfg, serviceLister, fdsPushRequests) - - if cfg.MeshPeers.Local.IngressType == config.OpenShiftRouter { - go resolveRemoteIP(ctx, cfg.MeshPeers.Remotes, meshConfigPushRequests) - } - importedServiceStore := fds.NewImportedServiceStore() for _, remote := range cfg.MeshPeers.Remotes { startFDSClient(ctx, remote, meshConfigPushRequests, importedServiceStore) } + informerFactory := informers.NewSharedInformerFactory(istioClient.Kube(), 0) + serviceLister := informerFactory.Core().V1().Services().Lister() + informerFactory.Start(ctx.Done()) startReconciler(ctx, cfg, serviceLister, meshConfigPushRequests, importedServiceStore) + + <-ctx.Done() } func startReconciler(ctx context.Context, cfg *config.Federation, serviceLister v1.ServiceLister, meshConfigPushRequests chan xds.PushRequest, importedServiceStore *fds.ImportedServiceStore) { @@ -239,26 +205,14 @@ func startReconciler(ctx context.Context, cfg *config.Federation, serviceLister istioConfigFactory := istio.NewConfigFactory(*cfg, serviceLister, importedServiceStore, namespace) reconcilers := []kube.Reconciler{ - kube.NewGatewayResourceReconciler(istioClient, istioConfigFactory), kube.NewServiceEntryReconciler(istioClient, istioConfigFactory), kube.NewWorkloadEntryReconciler(istioClient, istioConfigFactory), - kube.NewPeerAuthResourceReconciler(istioClient, namespace), } if cfg.MeshPeers.AnyRemotePeerWithOpenshiftRouterIngress() { reconcilers = append(reconcilers, kube.NewDestinationRuleReconciler(istioClient, istioConfigFactory)) } - if cfg.MeshPeers.Local.IngressType == config.OpenShiftRouter { - routeClient, err := routev1client.NewForConfig(kubeConfig) - if err != nil { - log.Fatalf("failed to create Route client: %v", err) - } - - reconcilers = append(reconcilers, kube.NewEnvoyFilterReconciler(istioClient, istioConfigFactory)) - reconcilers = append(reconcilers, kube.NewRouteReconciler(routeClient, openshift.NewConfigFactory(*cfg, serviceLister))) - } - rm := kube.NewReconcilerManager(meshConfigPushRequests, reconcilers...) if err := rm.ReconcileAll(ctx); err != nil { log.Fatalf("initial Istio resource reconciliation failed: %v", err) @@ -267,55 +221,6 @@ func startReconciler(ctx context.Context, cfg *config.Federation, serviceLister go rm.Start(ctx) } -func startFederationServer(ctx context.Context, cfg *config.Federation, serviceLister v1.ServiceLister, fdsPushRequests chan xds.PushRequest) { - federationServer := adss.NewServer( - fdsPushRequests, - fds.NewExportedServicesGenerator(*cfg, serviceLister), - ) - - go func() { - if err := federationServer.Run(ctx); err != nil { - log.Fatalf("failed to start FDS server: %v", err) - } - }() -} - -func resolveRemoteIP(ctx context.Context, remotes []config.Remote, meshConfigPushRequests chan xds.PushRequest) { - var prevIPs []string - for _, remote := range remotes { - prevIPs = append(prevIPs, networking.Resolve(remote.Addresses[0])...) - } - - resolveIPs := func() { - var currIPs []string - for _, remote := range remotes { - log.Debugf("Resolving %s", remote.Name) - currIPs = append(currIPs, networking.Resolve(remote.Addresses[0])...) - } - - sort.Strings(currIPs) - if !slices.Equal(prevIPs, currIPs) { - log.Infof("IP addresses have changed") - prevIPs = currIPs - meshConfigPushRequests <- xds.PushRequest{TypeUrl: xds.WorkloadEntryTypeUrl} - } - } - - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - -resolveLoop: - for { - select { - case <-ctx.Done(): - break resolveLoop - case <-ticker.C: - resolveIPs() - } - } - -} - func startFDSClient(ctx context.Context, remote config.Remote, meshConfigPushRequests chan xds.PushRequest, importedServiceStore *fds.ImportedServiceStore) { var discoveryAddr string if networking.IsIP(remote.Addresses[0]) { diff --git a/examples/README.md b/examples/README.md index 3e35709d..6124810a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -105,11 +105,11 @@ kwest apply -f examples/openshift/west-federation-ingress-gateway.yaml On KinD: ```shell WEST_GATEWAY_IP=$(kwest get svc federation-ingress-gateway -n istio-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -helm-east install east chart -n istio-system \ +helm-east upgrade --install east chart -n istio-system \ --values examples/kind/east-federation-controller.yaml \ --set "federation.meshPeers.remotes[0].addresses[0]=$WEST_GATEWAY_IP" EAST_GATEWAY_IP=$(keast get svc federation-ingress-gateway -n istio-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -helm-west install west chart -n istio-system \ +helm-west upgrade --install west chart -n istio-system \ --values examples/kind/west-federation-controller.yaml \ --set "federation.meshPeers.remotes[0].addresses[0]=$EAST_GATEWAY_IP" ``` @@ -117,13 +117,27 @@ helm-west install west chart -n istio-system \ On OpenShift: ```shell WEST_CONSOLE_URL=$(kwest get routes console -n openshift-console -o jsonpath='{.spec.host}') -helm-east install east chart -n istio-system --values examples/openshift/east-federation-controller.yaml \ +helm-east upgrade --install east chart -n istio-system --values examples/openshift/east-federation-controller.yaml \ --set "federation.meshPeers.remotes[0].addresses[0]=$WEST_CONSOLE_URL" EAST_CONSOLE_URL=$(keast get routes console -n openshift-console -o jsonpath='{.spec.host}') -helm-west install west chart -n istio-system --values examples/openshift/west-federation-controller.yaml \ +helm-west upgrade --install west chart -n istio-system --values examples/openshift/west-federation-controller.yaml \ --set "federation.meshPeers.remotes[0].addresses[0]=$EAST_CONSOLE_URL" ``` +### Enable mesh federation + +On KinD: +```shell +keast apply -f examples/kind/east-mesh-federation.yaml +kwest apply -f examples/kind/west-mesh-federation.yaml +``` + +On OpenShift: +```shell +keast apply -f examples/openshift/east-mesh-federation.yaml +kwest apply -f examples/openshift/west-mesh-federation.yaml +``` + ### Deploy and export services 1. Enable mTLS and deploy apps: @@ -156,12 +170,12 @@ keast delete sa bookinfo-ratings 3. Export services: ```shell -kwest label svc productpage export-service=true -kwest label svc reviews export-service=true -kwest label svc ratings export-service=true -keast label svc productpage export-service=true -keast label svc reviews export-service=true -keast label svc details export-service=true +kwest label svc productpage export=true +kwest label svc reviews export=true +kwest label svc ratings export=true +keast label svc productpage export=true +keast label svc reviews export=true +keast label svc details export=true ``` 4. Get gateway addresses: diff --git a/examples/kind/east-federation-controller.yaml b/examples/kind/east-federation-controller.yaml index 139d2820..a0aa1f9e 100644 --- a/examples/kind/east-federation-controller.yaml +++ b/examples/kind/east-federation-controller.yaml @@ -1,17 +1,7 @@ +image: + tag: latest-local federation: meshPeers: - local: - name: east - gateways: - ingress: - selector: - app: federation-ingress-gateway remotes: - name: west network: west-network - exportedServiceSet: - rules: - - type: LabelSelector - labelSelectors: - - matchLabels: - export-service: "true" diff --git a/examples/kind/east-mesh-federation.yaml b/examples/kind/east-mesh-federation.yaml new file mode 100644 index 00000000..cca158f3 --- /dev/null +++ b/examples/kind/east-mesh-federation.yaml @@ -0,0 +1,18 @@ +apiVersion: federation.openshift-service-mesh.io/v1alpha1 +kind: MeshFederation +metadata: + name: east + namespace: istio-system +spec: + ingress: + type: istio + gateway: + selector: + app: federation-ingress-gateway + portConfig: + name: tls-passthrough + number: 15443 + export: + serviceSelectors: + matchLabels: + export: "true" \ No newline at end of file diff --git a/examples/kind/west-federation-controller.yaml b/examples/kind/west-federation-controller.yaml index 3c6fec3e..93c187b2 100644 --- a/examples/kind/west-federation-controller.yaml +++ b/examples/kind/west-federation-controller.yaml @@ -1,17 +1,7 @@ +image: + tag: latest-local federation: meshPeers: - local: - name: west - gateways: - ingress: - selector: - app: federation-ingress-gateway remotes: - name: east network: east-network - exportedServiceSet: - rules: - - type: LabelSelector - labelSelectors: - - matchLabels: - export-service: "true" diff --git a/examples/kind/west-mesh-federation.yaml b/examples/kind/west-mesh-federation.yaml new file mode 100644 index 00000000..d711fb4d --- /dev/null +++ b/examples/kind/west-mesh-federation.yaml @@ -0,0 +1,18 @@ +apiVersion: federation.openshift-service-mesh.io/v1alpha1 +kind: MeshFederation +metadata: + name: west + namespace: istio-system +spec: + ingress: + type: istio + gateway: + selector: + app: federation-ingress-gateway + portConfig: + name: tls-passthrough + number: 15443 + export: + serviceSelectors: + matchLabels: + export: "true" diff --git a/examples/openshift/east-federation-controller.yaml b/examples/openshift/east-federation-controller.yaml index 2942d045..90e61398 100644 --- a/examples/openshift/east-federation-controller.yaml +++ b/examples/openshift/east-federation-controller.yaml @@ -1,20 +1,7 @@ federation: meshPeers: - local: - name: east - gateways: - ingress: - selector: - app: federation-ingress-gateway - ingressType: openshift-router remotes: - name: west ingressType: openshift-router port: 443 network: west-network - exportedServiceSet: - rules: - - type: LabelSelector - labelSelectors: - - matchLabels: - export-service: "true" diff --git a/examples/openshift/east-mesh-federation.yaml b/examples/openshift/east-mesh-federation.yaml new file mode 100644 index 00000000..34b372bb --- /dev/null +++ b/examples/openshift/east-mesh-federation.yaml @@ -0,0 +1,18 @@ +apiVersion: federation.openshift-service-mesh.io/v1alpha1 +kind: MeshFederation +metadata: + name: east + namespace: istio-system +spec: + ingress: + type: openshift-router + gateway: + selector: + app: federation-ingress-gateway + portConfig: + name: tls-passthrough + number: 15443 + export: + serviceSelectors: + matchLabels: + export: "true" diff --git a/examples/openshift/west-federation-controller.yaml b/examples/openshift/west-federation-controller.yaml index 92d45bb5..e2365666 100644 --- a/examples/openshift/west-federation-controller.yaml +++ b/examples/openshift/west-federation-controller.yaml @@ -1,20 +1,7 @@ federation: meshPeers: - local: - name: west - gateways: - ingress: - selector: - app: federation-ingress-gateway - ingressType: openshift-router remotes: - name: east ingressType: openshift-router port: 443 network: east-network - exportedServiceSet: - rules: - - type: LabelSelector - labelSelectors: - - matchLabels: - export-service: "true" diff --git a/examples/openshift/west-mesh-federation.yaml b/examples/openshift/west-mesh-federation.yaml new file mode 100644 index 00000000..54734b17 --- /dev/null +++ b/examples/openshift/west-mesh-federation.yaml @@ -0,0 +1,18 @@ +apiVersion: federation.openshift-service-mesh.io/v1alpha1 +kind: MeshFederation +metadata: + name: west + namespace: istio-system +spec: + ingress: + type: openshift-router + gateway: + selector: + app: federation-ingress-gateway + portConfig: + name: tls-passthrough + number: 15443 + export: + serviceSelectors: + matchLabels: + export: "true" diff --git a/go.mod b/go.mod index d6e424af..48b57de8 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,6 @@ replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.5 require ( github.com/envoyproxy/go-control-plane v0.12.1-0.20240415211714-57c85e1829e6 github.com/openshift/api v0.0.0-20240404200104-96ed2d49b255 - github.com/openshift/client-go v0.0.0-20231212205830-0ab0864ec8c2 - golang.org/x/net v0.27.0 golang.org/x/sync v0.7.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 @@ -185,6 +183,7 @@ require ( golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.22.0 // indirect diff --git a/go.sum b/go.sum index 76a49214..3b02e2a5 100644 --- a/go.sum +++ b/go.sum @@ -485,8 +485,6 @@ github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/ github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/openshift/api v0.0.0-20240404200104-96ed2d49b255 h1:OPEl/rl/Bt8soLkMUex9PZu9PJB59VPFnaPh/n1Pb3I= github.com/openshift/api v0.0.0-20240404200104-96ed2d49b255/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= -github.com/openshift/client-go v0.0.0-20231212205830-0ab0864ec8c2 h1:ArsCqZ2t7Jepm44YxW/4t2q1bPcqiyn5erNwpfbk8dE= -github.com/openshift/client-go v0.0.0-20231212205830-0ab0864ec8c2/go.mod h1:rk91ouw63QUVu2NfUt09MSJT4W54q5J5EV94f87jNC8= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/internal/controller/meshfederation/dns_resolver.go b/internal/controller/meshfederation/dns_resolver.go new file mode 100644 index 00000000..83ea525e --- /dev/null +++ b/internal/controller/meshfederation/dns_resolver.go @@ -0,0 +1,63 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the License); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meshfederation + +import ( + "context" + "sort" + "time" + + "istio.io/istio/pkg/slices" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/openshift-service-mesh/federation/internal/pkg/networking" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" +) + +func (r *Reconciler) resolveRemoteIP(ctx context.Context) { + logger := log.FromContext(ctx) + + var prevIPs []string + for _, remote := range r.remotes { + prevIPs = append(prevIPs, networking.Resolve(remote.Addresses[0])...) + } + + resolveIPs := func() { + var currIPs []string + for _, remote := range r.remotes { + logger.Info("Resolving address", "remote-mesh", remote.Name) + currIPs = append(currIPs, networking.Resolve(remote.Addresses[0])...) + } + + sort.Strings(currIPs) + if !slices.Equal(prevIPs, currIPs) { + prevIPs = currIPs + r.meshConfigPushRequests <- xds.PushRequest{TypeUrl: xds.WorkloadEntryTypeUrl} + } + } + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + +resolveLoop: + for { + select { + case <-ctx.Done(): + break resolveLoop + case <-ticker.C: + resolveIPs() + } + } +} diff --git a/internal/controller/meshfederation/envoy_filter.go b/internal/controller/meshfederation/envoy_filter.go new file mode 100644 index 00000000..ef80fff9 --- /dev/null +++ b/internal/controller/meshfederation/envoy_filter.go @@ -0,0 +1,95 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the License); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meshfederation + +import ( + "context" + "fmt" + + "google.golang.org/protobuf/types/known/structpb" + networkingspecv1alpha3 "istio.io/api/networking/v1alpha3" + "istio.io/client-go/pkg/apis/networking/v1alpha3" + "istio.io/istio/pkg/util/protomarshal" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func (r *Reconciler) reconcileEnvoyFilters(ctx context.Context, federatedServices *corev1.ServiceList) error { + if err := r.createOrUpdateEnvoyFilter(ctx, fmt.Sprintf("federation-discovery-service-%s", r.instance.Name), r.namespace, 15080); err != nil { + return err + } + for _, svc := range federatedServices.Items { + for _, port := range svc.Spec.Ports { + if err := r.createOrUpdateEnvoyFilter(ctx, svc.Name, svc.Namespace, port.Port); err != nil { + return err + } + } + } + return nil +} + +func (r *Reconciler) createOrUpdateEnvoyFilter(ctx context.Context, svcName, svcNs string, port int32) error { + envoyFilter := &v1alpha3.EnvoyFilter{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("sni-%s-%s-%d", svcName, svcNs, port), + Namespace: r.namespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, envoyFilter, func() error { + patchValue, err := buildPatchStruct(routerCompatibleSNI(svcName, svcNs, uint32(port))) + if err != nil { + return err + } + envoyFilter.Spec = networkingspecv1alpha3.EnvoyFilter{ + WorkloadSelector: &networkingspecv1alpha3.WorkloadSelector{ + Labels: r.instance.Spec.IngressConfig.GatewayConfig.Selector, + }, + ConfigPatches: []*networkingspecv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{{ + ApplyTo: networkingspecv1alpha3.EnvoyFilter_FILTER_CHAIN, + Match: &networkingspecv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + ObjectTypes: &networkingspecv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &networkingspecv1alpha3.EnvoyFilter_ListenerMatch{ + Name: fmt.Sprintf("0.0.0.0_%d", r.instance.Spec.IngressConfig.GatewayConfig.PortConfig.Number), + FilterChain: &networkingspecv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Sni: fmt.Sprintf("outbound_.%d_._.%s.%s.svc.cluster.local", port, svcName, svcNs), + }, + }, + }, + }, + Patch: &networkingspecv1alpha3.EnvoyFilter_Patch{ + Operation: networkingspecv1alpha3.EnvoyFilter_Patch_MERGE, + Value: patchValue, + }, + }}, + } + return controllerutil.SetControllerReference(r.instance, envoyFilter, r.Scheme()) + }) + return err +} + +func buildPatchStruct(sni string) (*structpb.Struct, error) { + patchConfig := fmt.Sprintf(`{"filter_chain_match":{"server_names":["%s"]}}`, sni) + serializedConfig := &structpb.Struct{} + if err := protomarshal.UnmarshalString(patchConfig, serializedConfig); err != nil { + return nil, fmt.Errorf("failed to unmarshal envoy filter patch config") + } + return serializedConfig, nil +} + +// routerCompatibleSNI returns SNI compatible with https://datatracker.ietf.org/doc/html/rfc952 required by OpenShift Router. +func routerCompatibleSNI(svcName, svcNs string, port uint32) string { + return fmt.Sprintf("%s-%d.%s.svc.cluster.local", svcName, port, svcNs) +} diff --git a/internal/controller/meshfederation/gateway.go b/internal/controller/meshfederation/gateway.go new file mode 100644 index 00000000..ae02cd49 --- /dev/null +++ b/internal/controller/meshfederation/gateway.go @@ -0,0 +1,58 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the License); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meshfederation + +import ( + "context" + "fmt" + + networkingspecv1alpha3 "istio.io/api/networking/v1alpha3" + networkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func (r *Reconciler) reconcileGateway(ctx context.Context, federatedServices *corev1.ServiceList) error { + hosts := []string{fmt.Sprintf("federation-discovery-service-%s.%s.svc.cluster.local", r.instance.Name, r.instance.Namespace)} + for _, svc := range federatedServices.Items { + hosts = append(hosts, fmt.Sprintf("%s.%s.svc.cluster.local", svc.Name, svc.Namespace)) + } + gateway := &networkingv1alpha3.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "federation-ingress-gateway", + Namespace: r.namespace, + }, + } + // TODO: do not update if not necessary + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, gateway, func() error { + gateway.Spec = networkingspecv1alpha3.Gateway{ + Selector: r.instance.Spec.IngressConfig.GatewayConfig.Selector, + Servers: []*networkingspecv1alpha3.Server{{ + Hosts: hosts, + Port: &networkingspecv1alpha3.Port{ + Number: r.instance.Spec.IngressConfig.GatewayConfig.PortConfig.Number, + Name: r.instance.Spec.IngressConfig.GatewayConfig.PortConfig.Name, + Protocol: "TLS", + }, + Tls: &networkingspecv1alpha3.ServerTLSSettings{ + Mode: networkingspecv1alpha3.ServerTLSSettings_AUTO_PASSTHROUGH, + }, + }}, + } + return controllerutil.SetControllerReference(r.instance, gateway, r.Scheme()) + }) + return err +} diff --git a/internal/controller/meshfederation/meshfederation_controller.go b/internal/controller/meshfederation/meshfederation_controller.go index 0fa912b3..df555b4f 100644 --- a/internal/controller/meshfederation/meshfederation_controller.go +++ b/internal/controller/meshfederation/meshfederation_controller.go @@ -17,11 +17,23 @@ package meshfederation import ( "context" + networkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3" + securityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/openshift-service-mesh/federation/api/v1alpha1" + "github.com/openshift-service-mesh/federation/internal/pkg/config" + "github.com/openshift-service-mesh/federation/internal/pkg/fds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds/adss" ) // +kubebuilder:rbac:groups=federation.openshift-service-mesh.io,resources=meshfederations,verbs=get;list;watch;create;update;patch;delete @@ -31,14 +43,110 @@ import ( // Reconciler ensure that cluster is configured according to the spec defined in MeshFederation object. type Reconciler struct { client.Client + namespace string + + fdsServer *adss.Server + serverCtx context.Context + pushRequests chan xds.PushRequest + + instance *v1alpha1.MeshFederation + + remotes []config.Remote + + dnsResolverCtx context.Context + meshConfigPushRequests chan xds.PushRequest } -func NewReconciler(c client.Client) *Reconciler { - return &Reconciler{Client: c} +func NewReconciler(c client.Client, remotes []config.Remote, meshConfigPushRequests chan xds.PushRequest) *Reconciler { + return &Reconciler{ + Client: c, + namespace: config.PodNamespace(), + remotes: remotes, + meshConfigPushRequests: meshConfigPushRequests, + } } func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log.FromContext(ctx).Info("Reconciling object", "namespace", req.Namespace) + logger := log.FromContext(ctx) + logger.Info("Reconciling object") + + instance := &v1alpha1.MeshFederation{} + if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { + if errors.IsNotFound(err) { + logger.Info("Object not found, must have been deleted", "name", req.Name, "namespace", req.Namespace) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if !instance.DeletionTimestamp.IsZero() { + logger.Info("Object is being deleted", "name", req.Name, "namespace", req.Namespace) + + // stop FDS server + if r.serverCtx != nil { + r.serverCtx.Done() + } + r.fdsServer = nil + close(r.pushRequests) + + // stop DNS reconciler + if r.dnsResolverCtx != nil { + r.dnsResolverCtx.Done() + } + + // TODO: Handle finalizer + + r.instance = nil + + return ctrl.Result{}, nil + } + + r.instance = instance + + if err := r.reconcilePeerAuthentication(ctx); err != nil { + logger.Error(err, "failed to reconcile peer authentication") + return ctrl.Result{}, err + } + + // Start FDS server + if r.fdsServer == nil { + r.pushRequests = make(chan xds.PushRequest) + r.fdsServer = adss.NewServer(r.pushRequests, fds.NewDiscoveryResponseGenerator(r.Client, instance.Spec.ExportRules.ServiceSelectors)) + r.serverCtx = context.Background() + // TODO: restart server if necessary + go func() { + if err := r.fdsServer.Run(r.serverCtx); err != nil { + log.FromContext(ctx).Error(err, "failed to run FDS server") + panic("failed to run FDS server") + } + }() + } + + federatedServices, svcErr := r.reconcileFederatedServices(ctx) + if svcErr != nil { + logger.Error(svcErr, "failed to reconcile federated services") + return ctrl.Result{}, svcErr + } + + if err := r.reconcileGateway(ctx, federatedServices); err != nil { + logger.Error(err, "failed to reconcile gateway") + return ctrl.Result{}, err + } + + if r.instance.Spec.IngressConfig.Type == string(config.OpenShiftRouter) { + if err := r.reconcileEnvoyFilters(ctx, federatedServices); err != nil { + logger.Error(err, "failed to reconcile envoy filters") + return ctrl.Result{}, err + } + if err := r.reconcileRoutes(ctx, federatedServices); err != nil { + logger.Error(err, "failed to reconcile routes") + return ctrl.Result{}, err + } + // TODO: We should run DNS resolver when ingress address is DNS as well, not only when remote ingress is OpenShift Router + r.dnsResolverCtx = context.Background() + go r.resolveRemoteIP(r.dnsResolverCtx) + } + return ctrl.Result{}, nil } @@ -46,5 +154,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.MeshFederation{}). + Owns(&securityv1beta1.PeerAuthentication{}). + Owns(&networkingv1alpha3.Gateway{}). + Watches( + &corev1.Service{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueRequestForCurrentInstance), + builder.WithPredicates(r.enqueueIfMatchExportRules()), + ). Complete(r) } + +func (r *Reconciler) enqueueRequestForCurrentInstance(_ context.Context, _ client.Object) []reconcile.Request { + if r.instance == nil { + return []reconcile.Request{} + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: r.instance.Name, + Namespace: r.instance.Namespace, + }}} +} diff --git a/internal/controller/meshfederation/peer_authentication.go b/internal/controller/meshfederation/peer_authentication.go new file mode 100644 index 00000000..34696c9a --- /dev/null +++ b/internal/controller/meshfederation/peer_authentication.go @@ -0,0 +1,48 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the License); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meshfederation + +import ( + "context" + + securityspecv1beta1 "istio.io/api/security/v1beta1" + typev1beta1 "istio.io/api/type/v1beta1" + securityv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func (r *Reconciler) reconcilePeerAuthentication(ctx context.Context) error { + peerAuth := &securityv1beta1.PeerAuthentication{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fds-strict-mtls", + Namespace: r.namespace, + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, peerAuth, func() error { + peerAuth.Spec = securityspecv1beta1.PeerAuthentication{ + Selector: &typev1beta1.WorkloadSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "federation-controller", + }, + }, + Mtls: &securityspecv1beta1.PeerAuthentication_MutualTLS{ + Mode: securityspecv1beta1.PeerAuthentication_MutualTLS_STRICT, + }, + } + return controllerutil.SetControllerReference(r.instance, peerAuth, r.Scheme()) + }) + return err +} diff --git a/internal/controller/meshfederation/route.go b/internal/controller/meshfederation/route.go new file mode 100644 index 00000000..fd10ed85 --- /dev/null +++ b/internal/controller/meshfederation/route.go @@ -0,0 +1,66 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the License); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meshfederation + +import ( + "context" + "fmt" + + routev1 "github.com/openshift/api/route/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func (r *Reconciler) reconcileRoutes(ctx context.Context, federatedServices *corev1.ServiceList) error { + if err := r.createOrUpdateRoute(ctx, fmt.Sprintf("federation-discovery-service-%s", r.instance.Name), r.namespace, 15080); err != nil { + return err + } + for _, svc := range federatedServices.Items { + for _, port := range svc.Spec.Ports { + if err := r.createOrUpdateRoute(ctx, svc.Name, svc.Namespace, uint32(port.Port)); err != nil { + return err + } + } + } + return nil +} + +func (r *Reconciler) createOrUpdateRoute(ctx context.Context, svcName, svcNs string, port uint32) error { + route := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%d-to-federation-ingress-gateway", svcName, svcNs, port), + Namespace: r.namespace, // TODO: this should come from spec.ingress.gateway.namespace + }, + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, route, func() error { + route.Spec = routev1.RouteSpec{ + Host: fmt.Sprintf("%s-%d.%s.svc.cluster.local", svcName, port, svcNs), + To: routev1.RouteTargetReference{ + Kind: "Service", + Name: "federation-ingress-gateway", // TODO: this name should be configurable + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString(r.instance.Spec.IngressConfig.GatewayConfig.PortConfig.Name), + }, + TLS: &routev1.TLSConfig{ + Termination: routev1.TLSTerminationPassthrough, + }, + } + return controllerutil.SetControllerReference(r.instance, route, r.Scheme()) + }) + return err +} diff --git a/internal/controller/meshfederation/service.go b/internal/controller/meshfederation/service.go new file mode 100644 index 00000000..116146bf --- /dev/null +++ b/internal/controller/meshfederation/service.go @@ -0,0 +1,75 @@ +// Copyright Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the License); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package meshfederation + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/openshift-service-mesh/federation/internal/pkg/xds" +) + +func (r *Reconciler) reconcileFederatedServices(ctx context.Context) (*corev1.ServiceList, error) { + exportedServices := &corev1.ServiceList{} + // TODO: Add support for matchExpressions + if err := r.Client.List(ctx, exportedServices, client.MatchingLabels(r.instance.Spec.ExportRules.ServiceSelectors.MatchLabels)); err != nil { + return nil, fmt.Errorf("failed to list services: %w", err) + } + + // TODO: Do not update if not necessary + r.pushRequests <- xds.PushRequest{TypeUrl: xds.ExportedServiceTypeUrl} + + return exportedServices, nil +} + +func (r *Reconciler) enqueueIfMatchExportRules() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return r.matchesExportRules(e.Object.(*corev1.Service)) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldSvc := e.ObjectOld.(*corev1.Service) + newSvc := e.ObjectNew.(*corev1.Service) + return r.matchesExportRules(oldSvc) != r.matchesExportRules(newSvc) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return r.matchesExportRules(e.Object.(*corev1.Service)) + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + } +} + +func (r *Reconciler) matchesExportRules(svc *corev1.Service) bool { + if r.instance == nil { + return false + } + if r.instance.Spec.ExportRules == nil { + return false + } + if r.instance.Spec.ExportRules.ServiceSelectors == nil { + return true + } + // TODO: add support for matchExpressions + selector := labels.SelectorFromSet(r.instance.Spec.ExportRules.ServiceSelectors.MatchLabels) + return selector.Matches(labels.Set(svc.GetLabels())) +} diff --git a/internal/pkg/legacy/fds/exported_service_generator.go b/internal/pkg/fds/federated_service_response_generator.go similarity index 53% rename from internal/pkg/legacy/fds/exported_service_generator.go rename to internal/pkg/fds/federated_service_response_generator.go index 736a28bc..d86f4ecd 100644 --- a/internal/pkg/legacy/fds/exported_service_generator.go +++ b/internal/pkg/fds/federated_service_response_generator.go @@ -15,68 +15,66 @@ package fds import ( + "context" "fmt" "strings" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "k8s.io/apimachinery/pkg/labels" - v1 "k8s.io/client-go/listers/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift-service-mesh/federation/internal/api/federation/v1alpha1" - "github.com/openshift-service-mesh/federation/internal/pkg/config" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds/adss" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds/adss" ) -var _ adss.RequestHandler = (*ExportedServicesGenerator)(nil) +var _ adss.RequestHandler = (*DiscoveryResponseGenerator)(nil) -type ExportedServicesGenerator struct { - cfg config.Federation - serviceLister v1.ServiceLister +type DiscoveryResponseGenerator struct { + c client.Client + serviceSelectors *metav1.LabelSelector } -func NewExportedServicesGenerator(cfg config.Federation, serviceLister v1.ServiceLister) *ExportedServicesGenerator { - return &ExportedServicesGenerator{ - cfg: cfg, - serviceLister: serviceLister, +func NewDiscoveryResponseGenerator(c client.Client, serviceSelectors *metav1.LabelSelector) *DiscoveryResponseGenerator { + return &DiscoveryResponseGenerator{ + c: c, + serviceSelectors: serviceSelectors, } } -func (g *ExportedServicesGenerator) GetTypeUrl() string { +func (f *DiscoveryResponseGenerator) GetTypeUrl() string { return xds.ExportedServiceTypeUrl } -func (g *ExportedServicesGenerator) GenerateResponse() ([]*anypb.Any, error) { - var exportedServices []*v1alpha1.FederatedService - for _, exportLabelSelector := range g.cfg.ExportedServiceSet.GetLabelSelectors() { - matchExported := labels.SelectorFromSet(exportLabelSelector.MatchLabels) - services, err := g.serviceLister.List(matchExported) - if err != nil { - return nil, fmt.Errorf("failed to list services: %w", err) - } - for _, svc := range services { - var ports []*v1alpha1.ServicePort - for _, port := range svc.Spec.Ports { - servicePort := &v1alpha1.ServicePort{ - Name: port.Name, - Number: uint32(port.Port), - } - if port.TargetPort.IntVal != 0 { - servicePort.TargetPort = uint32(port.TargetPort.IntVal) - } - servicePort.Protocol = detectProtocol(port.Name) - ports = append(ports, servicePort) +func (f *DiscoveryResponseGenerator) GenerateResponse() ([]*anypb.Any, error) { + var federatedServices []*v1alpha1.FederatedService + serviceList := &corev1.ServiceList{} + // TODO: Add support for matchExpressions + if err := f.c.List(context.Background(), serviceList, client.MatchingLabels(f.serviceSelectors.MatchLabels)); err != nil { + return nil, fmt.Errorf("failed to list services: %w", err) + } + for _, svc := range serviceList.Items { + var ports []*v1alpha1.ServicePort + for _, port := range svc.Spec.Ports { + servicePort := &v1alpha1.ServicePort{ + Name: port.Name, + Number: uint32(port.Port), } - exportedService := &v1alpha1.FederatedService{ - Hostname: fmt.Sprintf("%s.%s.svc.cluster.local", svc.Name, svc.Namespace), - Ports: ports, - Labels: svc.Labels, + if port.TargetPort.IntVal != 0 { + servicePort.TargetPort = uint32(port.TargetPort.IntVal) } - exportedServices = append(exportedServices, exportedService) + servicePort.Protocol = detectProtocol(port.Name) + ports = append(ports, servicePort) } + federatedServices = append(federatedServices, &v1alpha1.FederatedService{ + Hostname: fmt.Sprintf("%s.%s.svc.cluster.local", svc.Name, svc.Namespace), + Ports: ports, + Labels: svc.Labels, + }) } - return serialize(exportedServices) + return serialize(federatedServices) } // TODO: check appProtocol and reject UDP diff --git a/internal/pkg/istio/config_factory.go b/internal/pkg/istio/config_factory.go index ef40a223..cb93e670 100644 --- a/internal/pkg/istio/config_factory.go +++ b/internal/pkg/istio/config_factory.go @@ -16,19 +16,15 @@ package istio import ( "fmt" - "sort" "strings" - "google.golang.org/protobuf/types/known/structpb" istionetv1alpha3 "istio.io/api/networking/v1alpha3" "istio.io/client-go/pkg/apis/networking/v1alpha3" istiolog "istio.io/istio/pkg/log" "istio.io/istio/pkg/maps" "istio.io/istio/pkg/slices" - "istio.io/istio/pkg/util/protomarshal" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" v1 "k8s.io/client-go/listers/core/v1" "github.com/openshift-service-mesh/federation/internal/api/federation/v1alpha1" @@ -37,10 +33,6 @@ import ( "github.com/openshift-service-mesh/federation/internal/pkg/networking" ) -const ( - federationIngressGatewayName = "federation-ingress-gateway" -) - type ConfigFactory struct { cfg config.Federation serviceLister v1.ServiceLister @@ -131,113 +123,6 @@ func (cf *ConfigFactory) DestinationRules() []*v1alpha3.DestinationRule { return destinationRules } -func (cf *ConfigFactory) IngressGateway() (*v1alpha3.Gateway, error) { - gateway := &v1alpha3.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: federationIngressGatewayName, - Namespace: cf.cfg.MeshPeers.Local.ControlPlane.Namespace, - Labels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }, - Spec: istionetv1alpha3.Gateway{ - Selector: cf.cfg.MeshPeers.Local.Gateways.Ingress.Selector, - Servers: []*istionetv1alpha3.Server{{ - Hosts: []string{}, - Port: &istionetv1alpha3.Port{ - Number: cf.cfg.MeshPeers.Local.Gateways.Ingress.Port.Number, - Name: cf.cfg.MeshPeers.Local.Gateways.Ingress.Port.Name, - Protocol: "TLS", - }, - Tls: &istionetv1alpha3.ServerTLSSettings{ - Mode: istionetv1alpha3.ServerTLSSettings_AUTO_PASSTHROUGH, - }, - }}, - }, - } - - hosts := []string{fmt.Sprintf("federation-discovery-service-%s.%s.svc.cluster.local", cf.cfg.MeshPeers.Local.Name, cf.namespace)} - for _, exportLabelSelector := range cf.cfg.ExportedServiceSet.GetLabelSelectors() { - matchLabels := labels.SelectorFromSet(exportLabelSelector.MatchLabels) - services, err := cf.serviceLister.List(matchLabels) - if err != nil { - return nil, fmt.Errorf("error listing services (selector=%s): %w", matchLabels, err) - } - for _, svc := range services { - hosts = append(hosts, fmt.Sprintf("%s.%s.svc.cluster.local", svc.Name, svc.Namespace)) - } - } - // ServiceLister.List is not idempotent, so to avoid redundant XDS push from Istio to proxies, - // we must return hostnames in the same order. - sort.Strings(hosts) - gateway.Spec.Servers[0].Hosts = hosts - - return gateway, nil -} - -// EnvoyFilters returns patches for SNI filters matching SNIs of exported services in federation ingress gateway. -// These patches add SNI compatible with https://datatracker.ietf.org/doc/html/rfc952 required by OpenShift Router. -// This function returns nil when the local ingress type is "istio". -func (cf *ConfigFactory) EnvoyFilters() []*v1alpha3.EnvoyFilter { - if cf.cfg.MeshPeers.Local.IngressType != config.OpenShiftRouter { - return nil - } - - createEnvoyFilter := func(svcName, svcNamespace string, port int32) *v1alpha3.EnvoyFilter { - buildPatchStruct := func(config string) *structpb.Struct { - val := &structpb.Struct{} - if err := protomarshal.UnmarshalString(config, val); err != nil { - fmt.Printf("error unmarshalling envoyfilter config %q: %v", config, err) - } - return val - } - return &v1alpha3.EnvoyFilter{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("sni-%s-%s-%d", svcName, svcNamespace, port), - Namespace: cf.cfg.MeshPeers.Local.ControlPlane.Namespace, - Labels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }, - Spec: istionetv1alpha3.EnvoyFilter{ - WorkloadSelector: &istionetv1alpha3.WorkloadSelector{ - Labels: cf.cfg.MeshPeers.Local.Gateways.Ingress.Selector, - }, - ConfigPatches: []*istionetv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{{ - ApplyTo: istionetv1alpha3.EnvoyFilter_FILTER_CHAIN, - Match: &istionetv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ - ObjectTypes: &istionetv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ - Listener: &istionetv1alpha3.EnvoyFilter_ListenerMatch{ - Name: fmt.Sprintf("0.0.0.0_%d", cf.cfg.MeshPeers.Local.Gateways.Ingress.Port.Number), - FilterChain: &istionetv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ - Sni: fmt.Sprintf("outbound_.%d_._.%s.%s.svc.cluster.local", port, svcName, svcNamespace), - }, - }, - }, - }, - Patch: &istionetv1alpha3.EnvoyFilter_Patch{ - Operation: istionetv1alpha3.EnvoyFilter_Patch_MERGE, - Value: buildPatchStruct(fmt.Sprintf(`{"filter_chain_match":{"server_names":["%s"]}}`, routerCompatibleSNI(svcName, svcNamespace, uint32(port)))), - }, - }}, - }, - } - } - - envoyFilters := []*v1alpha3.EnvoyFilter{ - createEnvoyFilter(fmt.Sprintf("federation-discovery-service-%s", cf.cfg.MeshPeers.Local.Name), "istio-system", 15080), - } - for _, exportLabelSelector := range cf.cfg.ExportedServiceSet.GetLabelSelectors() { - matchLabels := labels.SelectorFromSet(exportLabelSelector.MatchLabels) - services, err := cf.serviceLister.List(matchLabels) - if err != nil { - fmt.Printf("error listing services (selector=%s): %v", matchLabels, err) - } - for _, svc := range services { - for _, port := range svc.Spec.Ports { - envoyFilters = append(envoyFilters, createEnvoyFilter(svc.Name, svc.Namespace, port.Port)) - } - } - } - return envoyFilters -} - func (cf *ConfigFactory) ServiceEntries() ([]*v1alpha3.ServiceEntry, error) { var serviceEntries []*v1alpha3.ServiceEntry diff --git a/internal/pkg/istio/config_factory_test.go b/internal/pkg/istio/config_factory_test.go index 669ae648..f85175c6 100644 --- a/internal/pkg/istio/config_factory_test.go +++ b/internal/pkg/istio/config_factory_test.go @@ -19,11 +19,8 @@ import ( "encoding/json" "os" "path/filepath" - "reflect" "testing" - istionetv1alpha3 "istio.io/api/networking/v1alpha3" - "istio.io/client-go/pkg/apis/networking/v1alpha3" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -142,152 +139,6 @@ var ( } ) -func TestIngressGateway(t *testing.T) { - testCases := []struct { - name string - localServices []*corev1.Service - expectedGateway *v1alpha3.Gateway - }{{ - name: "federation-ingress-gateway should expose FDS and exported services", - localServices: []*corev1.Service{svcA_ns1, export(svcB_ns1), export(svcA_ns2)}, - expectedGateway: &v1alpha3.Gateway{ - ObjectMeta: v1.ObjectMeta{ - Name: "federation-ingress-gateway", - Namespace: "istio-system", - Labels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }, - Spec: istionetv1alpha3.Gateway{ - Selector: map[string]string{"app": "federation-ingress-gateway"}, - Servers: []*istionetv1alpha3.Server{{ - Hosts: []string{ - "a.ns2.svc.cluster.local", - "b.ns1.svc.cluster.local", - "federation-discovery-service-east.istio-system.svc.cluster.local", - }, - Port: &istionetv1alpha3.Port{ - Number: 443, - Name: "tls", - Protocol: "TLS", - }, - Tls: &istionetv1alpha3.ServerTLSSettings{ - Mode: istionetv1alpha3.ServerTLSSettings_AUTO_PASSTHROUGH, - }, - }}, - }, - }, - }, { - name: "federation-ingress-gateway should always expose FDS", - localServices: []*corev1.Service{}, - expectedGateway: &v1alpha3.Gateway{ - ObjectMeta: v1.ObjectMeta{ - Name: "federation-ingress-gateway", - Namespace: "istio-system", - Labels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }, - Spec: istionetv1alpha3.Gateway{ - Selector: map[string]string{"app": "federation-ingress-gateway"}, - Servers: []*istionetv1alpha3.Server{{ - Hosts: []string{ - "federation-discovery-service-east.istio-system.svc.cluster.local", - }, - Port: &istionetv1alpha3.Port{ - Number: 443, - Name: "tls", - Protocol: "TLS", - }, - Tls: &istionetv1alpha3.ServerTLSSettings{ - Mode: istionetv1alpha3.ServerTLSSettings_AUTO_PASSTHROUGH, - }, - }}, - }, - }, - }} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset() - informerFactory := informers.NewSharedInformerFactory(client, 0) - serviceInformer := informerFactory.Core().V1().Services().Informer() - serviceLister := informerFactory.Core().V1().Services().Lister() - stopCh := make(chan struct{}) - informerFactory.Start(stopCh) - - for _, svc := range tc.localServices { - if _, err := client.CoreV1().Services(svc.Namespace).Create(context.Background(), svc, v1.CreateOptions{}); err != nil { - t.Fatalf("failed to create service %s/%s: %v", svc.Name, svc.Namespace, err) - } - } - - serviceController, err := informer.NewResourceController(serviceInformer, corev1.Service{}) - if err != nil { - t.Fatalf("error creating serviceController: %v", err) - } - serviceController.RunAndWait(stopCh) - - factory := NewConfigFactory(exportConfig, serviceLister, fds.NewImportedServiceStore(), "istio-system") - actual, err := factory.IngressGateway() - if err != nil { - t.Errorf("got unexpected error: %s", err) - } - if !reflect.DeepEqual(actual, tc.expectedGateway) { - t.Errorf("got unexpected result:\nexpected:\n%v\ngot:\n%v", tc.expectedGateway, actual) - } - }) - } -} - -func TestEnvoyFilters(t *testing.T) { - testCases := []struct { - name string - localIngressType config.IngressType - localServices []*corev1.Service - expectedEnvoyFilterFiles []string - }{{ - name: "EnvoyFilters should not return filters when local ingress type is istio", - localIngressType: config.Istio, - localServices: []*corev1.Service{export(svcA_ns1), svcB_ns1}, - expectedEnvoyFilterFiles: []string{}, - }, { - name: "EnvoyFilters should return filters for exported services and FDS", - localIngressType: config.OpenShiftRouter, - localServices: []*corev1.Service{svcA_ns1, export(svcB_ns1), export(svcA_ns2)}, - expectedEnvoyFilterFiles: []string{"fds.yaml", "svc-b-ns-1-port-80.yaml", "svc-b-ns-1-port-443.yaml", "svc-a-ns-2.yaml"}, - }, { - name: "EnvoyFilters should return a filter for FDS even if no service is exported", - localIngressType: config.OpenShiftRouter, - localServices: []*corev1.Service{}, - expectedEnvoyFilterFiles: []string{"fds.yaml"}, - }} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset() - informerFactory := informers.NewSharedInformerFactory(client, 0) - serviceInformer := informerFactory.Core().V1().Services().Informer() - serviceLister := informerFactory.Core().V1().Services().Lister() - stopCh := make(chan struct{}) - informerFactory.Start(stopCh) - - for _, svc := range tc.localServices { - if _, err := client.CoreV1().Services(svc.Namespace).Create(context.Background(), svc, v1.CreateOptions{}); err != nil { - t.Fatalf("failed to create service %s/%s: %v", svc.Name, svc.Namespace, err) - } - } - - serviceController, err := informer.NewResourceController(serviceInformer, corev1.Service{}) - if err != nil { - t.Fatalf("error creating serviceController: %v", err) - } - serviceController.RunAndWait(stopCh) - - cfg := copyConfig(&exportConfig) - cfg.MeshPeers.Local.IngressType = tc.localIngressType - - factory := NewConfigFactory(*cfg, serviceLister, fds.NewImportedServiceStore(), "istio-system") - envoyFilters := factory.EnvoyFilters() - compareResources(t, "envoy-filters", tc.expectedEnvoyFilterFiles, envoyFilters) - }) - } -} - func TestServiceEntries(t *testing.T) { importConfigRemoteIP := copyConfig(&exportConfig) importConfigRemoteIP.MeshPeers.Remotes = []config.Remote{{ diff --git a/internal/pkg/istio/testdata/envoy-filters/fds.yaml b/internal/pkg/istio/testdata/envoy-filters/fds.yaml deleted file mode 100644 index 17885276..00000000 --- a/internal/pkg/istio/testdata/envoy-filters/fds.yaml +++ /dev/null @@ -1,22 +0,0 @@ -metadata: - name: sni-federation-discovery-service-east-istio-system-15080 - namespace: istio-system - labels: - federation.openshift-service-mesh.io/peer: todo -spec: - workloadSelector: - labels: - app: federation-ingress-gateway - configPatches: - - applyTo: FILTER_CHAIN - match: - listener: - name: "0.0.0.0_443" - filterChain: - sni: "outbound_.15080_._.federation-discovery-service-east.istio-system.svc.cluster.local" - patch: - operation: MERGE - value: - filter_chain_match: - server_names: - - "federation-discovery-service-east-15080.istio-system.svc.cluster.local" diff --git a/internal/pkg/istio/testdata/envoy-filters/svc-a-ns-2.yaml b/internal/pkg/istio/testdata/envoy-filters/svc-a-ns-2.yaml deleted file mode 100644 index 9f41ece5..00000000 --- a/internal/pkg/istio/testdata/envoy-filters/svc-a-ns-2.yaml +++ /dev/null @@ -1,22 +0,0 @@ -metadata: - name: sni-a-ns2-80 - namespace: istio-system - labels: - federation.openshift-service-mesh.io/peer: todo -spec: - workloadSelector: - labels: - app: federation-ingress-gateway - configPatches: - - applyTo: FILTER_CHAIN - match: - listener: - name: "0.0.0.0_443" - filterChain: - sni: "outbound_.80_._.a.ns2.svc.cluster.local" - patch: - operation: MERGE - value: - filter_chain_match: - server_names: - - "a-80.ns2.svc.cluster.local" diff --git a/internal/pkg/istio/testdata/envoy-filters/svc-b-ns-1-port-443.yaml b/internal/pkg/istio/testdata/envoy-filters/svc-b-ns-1-port-443.yaml deleted file mode 100644 index 66261b93..00000000 --- a/internal/pkg/istio/testdata/envoy-filters/svc-b-ns-1-port-443.yaml +++ /dev/null @@ -1,22 +0,0 @@ -metadata: - name: sni-b-ns1-443 - namespace: istio-system - labels: - federation.openshift-service-mesh.io/peer: todo -spec: - workloadSelector: - labels: - app: federation-ingress-gateway - configPatches: - - applyTo: FILTER_CHAIN - match: - listener: - name: "0.0.0.0_443" - filterChain: - sni: "outbound_.443_._.b.ns1.svc.cluster.local" - patch: - operation: MERGE - value: - filter_chain_match: - server_names: - - "b-443.ns1.svc.cluster.local" diff --git a/internal/pkg/istio/testdata/envoy-filters/svc-b-ns-1-port-80.yaml b/internal/pkg/istio/testdata/envoy-filters/svc-b-ns-1-port-80.yaml deleted file mode 100644 index 280ffa7e..00000000 --- a/internal/pkg/istio/testdata/envoy-filters/svc-b-ns-1-port-80.yaml +++ /dev/null @@ -1,22 +0,0 @@ -metadata: - name: sni-b-ns1-80 - namespace: istio-system - labels: - federation.openshift-service-mesh.io/peer: todo -spec: - workloadSelector: - labels: - app: federation-ingress-gateway - configPatches: - - applyTo: FILTER_CHAIN - match: - listener: - name: "0.0.0.0_443" - filterChain: - sni: "outbound_.80_._.b.ns1.svc.cluster.local" - patch: - operation: MERGE - value: - filter_chain_match: - server_names: - - "b-80.ns1.svc.cluster.local" diff --git a/internal/pkg/legacy/fds/exported_service_generator_test.go b/internal/pkg/legacy/fds/exported_service_generator_test.go deleted file mode 100644 index 37ab8ee6..00000000 --- a/internal/pkg/legacy/fds/exported_service_generator_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the License); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an AS IS BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fds - -import ( - "reflect" - "testing" - - "golang.org/x/net/context" - "google.golang.org/protobuf/types/known/anypb" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes/fake" - - "github.com/openshift-service-mesh/federation/internal/api/federation/v1alpha1" - "github.com/openshift-service-mesh/federation/internal/pkg/config" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/informer" -) - -var ( - federationConfig = config.Federation{ - MeshPeers: config.MeshPeers{ - Local: config.Local{ - Name: "cluster-local", - }, - }, - ExportedServiceSet: config.ExportedServiceSet{ - Rules: []config.Rules{{ - Type: "LabelSelector", - LabelSelectors: []config.LabelSelectors{{ - MatchLabels: map[string]string{ - "export": "true", - }, - }}, - }}, - }, - } - - allPorts = []corev1.ServicePort{ - {Name: "http", Port: 80, Protocol: "HTTP"}, - {Name: "http-prefix", Port: 81, Protocol: "HTTP"}, - {Name: "http2", Port: 82, Protocol: "HTTP"}, - {Name: "http2-prefix", Port: 83, Protocol: "HTTP"}, - {Name: "https", Port: 443, Protocol: "HTTPS"}, - {Name: "https-prefix", Port: 543, Protocol: "HTTPS"}, - {Name: "grpc", Port: 643, Protocol: "GRPC"}, - {Name: "grpc-prefix", Port: 743, Protocol: "GRPC"}, - {Name: "tls", Port: 843, Protocol: "TLS"}, - {Name: "tls-prefix", Port: 943, Protocol: "TLS"}, - {Name: "tcp", Port: 22, Protocol: "TCP"}, - {Name: "tcp-prefix", Port: 23, Protocol: "TCP"}, - {Name: "mongo", Port: 27017, Protocol: "MONGO"}, - {Name: "mongo-prefix", Port: 37017, Protocol: "MONGO"}, - {Name: "unknown", Port: 1, Protocol: "TCP"}, - } - allExportedPorts = []*v1alpha1.ServicePort{ - {Name: "http", Number: 80, Protocol: "HTTP"}, - {Name: "http-prefix", Number: 81, Protocol: "HTTP"}, - {Name: "http2", Number: 82, Protocol: "HTTP2"}, - {Name: "http2-prefix", Number: 83, Protocol: "HTTP2"}, - {Name: "https", Number: 443, Protocol: "HTTPS"}, - {Name: "https-prefix", Number: 543, Protocol: "HTTPS"}, - {Name: "grpc", Number: 643, Protocol: "GRPC"}, - {Name: "grpc-prefix", Number: 743, Protocol: "GRPC"}, - {Name: "tls", Number: 843, Protocol: "TLS"}, - {Name: "tls-prefix", Number: 943, Protocol: "TLS"}, - {Name: "tcp", Number: 22, Protocol: "TCP"}, - {Name: "tcp-prefix", Number: 23, Protocol: "TCP"}, - {Name: "mongo", Number: 27017, Protocol: "MONGO"}, - {Name: "mongo-prefix", Number: 37017, Protocol: "MONGO"}, - {Name: "unknown", Number: 1, Protocol: "TCP"}, - } -) - -func TestNewExportedServicesGenerator(t *testing.T) { - testCases := []struct { - name string - existingServices []*corev1.Service - expectedExportedServices []*v1alpha1.FederatedService - }{{ - name: "found 2 services matching configured label selector", - existingServices: []*corev1.Service{{ - ObjectMeta: v1.ObjectMeta{ - Name: "a", - Namespace: "ns1", - Labels: map[string]string{ - "app": "a", - }, - }, - }, { - ObjectMeta: v1.ObjectMeta{ - Name: "b", - Namespace: "ns1", - Labels: map[string]string{ - "app": "b", - "export": "true", - }, - }, - Spec: corev1.ServiceSpec{Ports: allPorts}, - }, { - ObjectMeta: v1.ObjectMeta{ - Name: "a", - Namespace: "ns2", - Labels: map[string]string{ - "app": "a", - "export": "true", - }, - }, - Spec: corev1.ServiceSpec{Ports: allPorts}, - }}, - expectedExportedServices: []*v1alpha1.FederatedService{{ - Hostname: "b.ns1.svc.cluster.local", - Ports: allExportedPorts, - Labels: map[string]string{ - "app": "b", - "export": "true", - }, - }, { - Hostname: "a.ns2.svc.cluster.local", - Ports: allExportedPorts, - Labels: map[string]string{ - "app": "a", - "export": "true", - }, - }}, - }} - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset() - informerFactory := informers.NewSharedInformerFactory(client, 0) - serviceInformer := informerFactory.Core().V1().Services().Informer() - serviceLister := informerFactory.Core().V1().Services().Lister() - stopCh := make(chan struct{}) - informerFactory.Start(stopCh) - - for _, svc := range tc.existingServices { - if _, err := client.CoreV1().Services(svc.Namespace).Create(context.Background(), svc, v1.CreateOptions{}); err != nil { - t.Fatalf("failed to create service %s/%s: %v", svc.Name, svc.Namespace, err) - } - } - - serviceController, err := informer.NewResourceController(serviceInformer, corev1.Service{}) - if err != nil { - t.Fatalf("error creating serviceController: %v", err) - } - serviceController.RunAndWait(stopCh) - - generator := NewExportedServicesGenerator(federationConfig, serviceLister) - - resources, err := generator.GenerateResponse() - if err != nil { - t.Fatalf("error generating response: %v", err) - } - exportedServices := deserializeExportedServices(t, resources) - if len(exportedServices) != len(tc.expectedExportedServices) { - t.Errorf("expected %d exported services but got %d", len(tc.expectedExportedServices), len(exportedServices)) - } - for idx, cfg := range exportedServices { - var found bool - // ExportedServiceGenerator utilizes cache.SharedIndexInformer.GetStore().List() that is not idempotent, - // because it does not sort Services, so we can't compare cfg with tc.expectedExportedServices[idx] - for _, expectedCfg := range tc.expectedExportedServices { - if reflect.DeepEqual(cfg.DeepCopy(), expectedCfg.DeepCopy()) { - found = true - break - } - } - if !found { - t.Errorf("did not find expected object: \n[%v], \ngot: \n[%v]", cfg, tc.expectedExportedServices[idx]) - } - } - }) - } -} - -func deserializeExportedServices(t *testing.T, resources []*anypb.Any) []*v1alpha1.FederatedService { - t.Helper() - var out []*v1alpha1.FederatedService - for _, res := range resources { - var exportedService v1alpha1.FederatedService - if err := res.UnmarshalTo(&exportedService); err != nil { - t.Errorf("failed to deserialize XDS resource: %v", err) - } - out = append(out, &exportedService) - } - return out -} diff --git a/internal/pkg/legacy/fds/import_handler.go b/internal/pkg/legacy/fds/import_handler.go index 9f57c4a5..58c5bb2b 100644 --- a/internal/pkg/legacy/fds/import_handler.go +++ b/internal/pkg/legacy/fds/import_handler.go @@ -21,8 +21,8 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/openshift-service-mesh/federation/internal/api/federation/v1alpha1" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds/adsc" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds/adsc" ) var _ adsc.ResponseHandler = (*ImportedServiceHandler)(nil) diff --git a/internal/pkg/legacy/informer/service_export_event_handler.go b/internal/pkg/legacy/informer/service_export_event_handler.go index 2800c486..871304d6 100644 --- a/internal/pkg/legacy/informer/service_export_event_handler.go +++ b/internal/pkg/legacy/informer/service_export_event_handler.go @@ -20,7 +20,7 @@ import ( "github.com/openshift-service-mesh/federation/internal/pkg/common" "github.com/openshift-service-mesh/federation/internal/pkg/config" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var _ Handler = (*ServiceExportEventHandler)(nil) diff --git a/internal/pkg/legacy/informer/service_export_event_handler_test.go b/internal/pkg/legacy/informer/service_export_event_handler_test.go index 5fe677f5..bf74c9af 100644 --- a/internal/pkg/legacy/informer/service_export_event_handler_test.go +++ b/internal/pkg/legacy/informer/service_export_event_handler_test.go @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/openshift-service-mesh/federation/internal/pkg/config" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var ( diff --git a/internal/pkg/legacy/kube/destination_rule_reconciler.go b/internal/pkg/legacy/kube/destination_rule_reconciler.go index 6beaed22..6e668f72 100644 --- a/internal/pkg/legacy/kube/destination_rule_reconciler.go +++ b/internal/pkg/legacy/kube/destination_rule_reconciler.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift-service-mesh/federation/internal/pkg/istio" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var _ Reconciler = (*DestinationRuleReconciler)(nil) diff --git a/internal/pkg/legacy/kube/envoy_filter_reconciler.go b/internal/pkg/legacy/kube/envoy_filter_reconciler.go deleted file mode 100644 index 1496ff91..00000000 --- a/internal/pkg/legacy/kube/envoy_filter_reconciler.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the License); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an AS IS BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kube - -import ( - "context" - "fmt" - "reflect" - - "istio.io/client-go/pkg/apis/networking/v1alpha3" - applyconfigurationv1 "istio.io/client-go/pkg/applyconfiguration/meta/v1" - applyv1alpha3 "istio.io/client-go/pkg/applyconfiguration/networking/v1alpha3" - "istio.io/istio/pkg/kube" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openshift-service-mesh/federation/internal/pkg/istio" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" -) - -var _ Reconciler = (*EnvoyFilterReconciler)(nil) - -type EnvoyFilterReconciler struct { - client kube.Client - cf *istio.ConfigFactory -} - -func NewEnvoyFilterReconciler(client kube.Client, cf *istio.ConfigFactory) *EnvoyFilterReconciler { - return &EnvoyFilterReconciler{ - client: client, - cf: cf, - } -} - -func (r *EnvoyFilterReconciler) GetTypeUrl() string { - return xds.EnvoyFilterTypeUrl -} - -func (r *EnvoyFilterReconciler) Reconcile(ctx context.Context) error { - envoyFilters := r.cf.EnvoyFilters() - if len(envoyFilters) == 0 { - return nil - } - - envoyFiltersMap := make(map[types.NamespacedName]*v1alpha3.EnvoyFilter, len(envoyFilters)) - for _, ef := range envoyFilters { - envoyFiltersMap[types.NamespacedName{Namespace: ef.Namespace, Name: ef.Name}] = ef - } - - oldEnvoyFilters, err := r.client.Istio().NetworkingV1alpha3().EnvoyFilters(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ - LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{ - MatchLabels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }), - }) - if err != nil { - return fmt.Errorf("failed to list envoy filters: %w", err) - } - oldEnvoyFiltersMap := make(map[types.NamespacedName]*v1alpha3.EnvoyFilter, len(oldEnvoyFilters.Items)) - for _, ef := range oldEnvoyFilters.Items { - oldEnvoyFiltersMap[types.NamespacedName{Namespace: ef.Namespace, Name: ef.Name}] = ef - } - - kind := "EnvoyFilter" - apiVersion := "networking.istio.io/v1alpha3" - for k, ef := range envoyFiltersMap { - oldEF, ok := oldEnvoyFiltersMap[k] - if !ok || !reflect.DeepEqual(&oldEF.Spec, &ef.Spec) { - // Envoy filter does not currently exist or requires update - newEF, err := r.client.Istio().NetworkingV1alpha3().EnvoyFilters(ef.GetNamespace()).Apply(ctx, - &applyv1alpha3.EnvoyFilterApplyConfiguration{ - TypeMetaApplyConfiguration: applyconfigurationv1.TypeMetaApplyConfiguration{ - Kind: &kind, - APIVersion: &apiVersion, - }, - ObjectMetaApplyConfiguration: &applyconfigurationv1.ObjectMetaApplyConfiguration{ - Name: &ef.Name, - Namespace: &ef.Namespace, - Labels: ef.Labels, - }, - Spec: &ef.Spec, - }, - metav1.ApplyOptions{ - TypeMeta: metav1.TypeMeta{ - Kind: kind, - APIVersion: apiVersion, - }, - Force: true, - FieldManager: "federation-controller", - }, - ) - if err != nil { - return fmt.Errorf("failed to apply envoy filter: %w", err) - } - log.Infof("Applied envoy filter: %v", newEF) - } - } - - for k, oldEF := range oldEnvoyFiltersMap { - if _, ok := envoyFiltersMap[k]; !ok { - err := r.client.Istio().NetworkingV1alpha3().EnvoyFilters(oldEF.GetNamespace()).Delete(ctx, oldEF.GetName(), metav1.DeleteOptions{}) - if client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to delete old envoy filter: %w", err) - } - log.Infof("Deleted envoy filter: %v", oldEF) - } - } - - return nil -} diff --git a/internal/pkg/legacy/kube/gateway_reconciler.go b/internal/pkg/legacy/kube/gateway_reconciler.go deleted file mode 100644 index 0b65699c..00000000 --- a/internal/pkg/legacy/kube/gateway_reconciler.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the License); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an AS IS BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kube - -import ( - "context" - "fmt" - - applyconfigurationv1 "istio.io/client-go/pkg/applyconfiguration/meta/v1" - networkingv1alpha3 "istio.io/client-go/pkg/applyconfiguration/networking/v1alpha3" - "istio.io/istio/pkg/kube" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openshift-service-mesh/federation/internal/pkg/istio" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" -) - -var _ Reconciler = (*GatewayResourceReconciler)(nil) - -type GatewayResourceReconciler struct { - client kube.Client - cf *istio.ConfigFactory -} - -func NewGatewayResourceReconciler(client kube.Client, cf *istio.ConfigFactory) *GatewayResourceReconciler { - return &GatewayResourceReconciler{ - client: client, - cf: cf, - } -} - -func (r *GatewayResourceReconciler) GetTypeUrl() string { - return xds.GatewayTypeUrl -} - -func (r *GatewayResourceReconciler) Reconcile(ctx context.Context) error { - gw, err := r.cf.IngressGateway() - if err != nil { - return fmt.Errorf("error generating ingress gateway: %w", err) - } - - kind := "Gateway" - apiVersion := "networking.istio.io/v1alpha3" - newGW, err := r.client.Istio().NetworkingV1alpha3().Gateways(gw.GetNamespace()).Apply(ctx, &networkingv1alpha3.GatewayApplyConfiguration{ - TypeMetaApplyConfiguration: applyconfigurationv1.TypeMetaApplyConfiguration{ - Kind: &kind, - APIVersion: &apiVersion, - }, - ObjectMetaApplyConfiguration: &applyconfigurationv1.ObjectMetaApplyConfiguration{ - Name: &gw.Name, - Namespace: &gw.Namespace, - Labels: gw.Labels, - }, - Spec: &gw.Spec, - Status: nil, - }, metav1.ApplyOptions{ - TypeMeta: metav1.TypeMeta{ - Kind: kind, - APIVersion: apiVersion, - }, - Force: true, - FieldManager: "federation-controller", - }) - if err != nil { - return fmt.Errorf("error applying ingress gateway: %w", err) - } - log.Infof("Applied ingress gateway: %v", newGW) - - return nil -} diff --git a/internal/pkg/legacy/kube/peer_authentication_reconciler.go b/internal/pkg/legacy/kube/peer_authentication_reconciler.go deleted file mode 100644 index 0b8c73de..00000000 --- a/internal/pkg/legacy/kube/peer_authentication_reconciler.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the License); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an AS IS BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kube - -import ( - "context" - "fmt" - - securityv1beta1 "istio.io/api/security/v1beta1" - typev1beta1 "istio.io/api/type/v1beta1" - "istio.io/client-go/pkg/apis/security/v1beta1" - applyconfigurationv1 "istio.io/client-go/pkg/applyconfiguration/meta/v1" - applyv1beta "istio.io/client-go/pkg/applyconfiguration/security/v1beta1" - "istio.io/istio/pkg/kube" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" -) - -var _ Reconciler = (*PeerAuthResourceReconciler)(nil) - -type PeerAuthResourceReconciler struct { - client kube.Client - namespace string -} - -func NewPeerAuthResourceReconciler(client kube.Client, namespace string) *PeerAuthResourceReconciler { - return &PeerAuthResourceReconciler{ - client: client, - namespace: namespace, - } -} - -func (r *PeerAuthResourceReconciler) GetTypeUrl() string { - return xds.PeerAuthenticationTypeUrl -} - -func (r *PeerAuthResourceReconciler) Reconcile(ctx context.Context) error { - pa := &v1beta1.PeerAuthentication{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fds-strict-mtls", - Namespace: r.namespace, - Labels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }, - Spec: securityv1beta1.PeerAuthentication{ - Selector: &typev1beta1.WorkloadSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/name": "federation-controller", - }, - }, - Mtls: &securityv1beta1.PeerAuthentication_MutualTLS{ - Mode: securityv1beta1.PeerAuthentication_MutualTLS_STRICT, - }, - }, - } - - kind := "PeerAuthentication" - apiVersion := "security.istio.io/v1beta1" - newPA, err := r.client.Istio().SecurityV1beta1().PeerAuthentications(pa.GetNamespace()).Apply(ctx, &applyv1beta.PeerAuthenticationApplyConfiguration{ - TypeMetaApplyConfiguration: applyconfigurationv1.TypeMetaApplyConfiguration{ - Kind: &kind, - APIVersion: &apiVersion, - }, - ObjectMetaApplyConfiguration: &applyconfigurationv1.ObjectMetaApplyConfiguration{ - Name: &pa.Name, - Namespace: &pa.Namespace, - Labels: pa.Labels, - }, - Spec: &pa.Spec, - }, metav1.ApplyOptions{ - TypeMeta: metav1.TypeMeta{ - Kind: kind, - APIVersion: apiVersion, - }, - Force: true, - FieldManager: "federation-controller", - }) - if err != nil { - return fmt.Errorf("error applying peer authentication: %w", err) - } - log.Infof("Applied peer authentication: %v", newPA) - - return nil -} diff --git a/internal/pkg/legacy/kube/reconciler_manager.go b/internal/pkg/legacy/kube/reconciler_manager.go index a93d915b..ff511c28 100644 --- a/internal/pkg/legacy/kube/reconciler_manager.go +++ b/internal/pkg/legacy/kube/reconciler_manager.go @@ -20,7 +20,7 @@ import ( istiolog "istio.io/istio/pkg/log" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var log = istiolog.RegisterScope("kube", "Kubernetes reconciler") diff --git a/internal/pkg/legacy/kube/route_reconciler.go b/internal/pkg/legacy/kube/route_reconciler.go deleted file mode 100644 index 509cacac..00000000 --- a/internal/pkg/legacy/kube/route_reconciler.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the License); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an AS IS BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kube - -import ( - "context" - "fmt" - "reflect" - - routev1 "github.com/openshift/api/route/v1" - routev1apply "github.com/openshift/client-go/route/applyconfigurations/route/v1" - "github.com/openshift/client-go/route/clientset/versioned" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" - "github.com/openshift-service-mesh/federation/internal/pkg/openshift" -) - -var _ Reconciler = (*RouteReconciler)(nil) - -type RouteReconciler struct { - client versioned.Interface - cf *openshift.ConfigFactory -} - -func NewRouteReconciler(client versioned.Interface, cf *openshift.ConfigFactory) *RouteReconciler { - return &RouteReconciler{ - client: client, - cf: cf, - } -} - -func (r *RouteReconciler) GetTypeUrl() string { - return xds.RouteTypeUrl -} - -func (r *RouteReconciler) Reconcile(ctx context.Context) error { - routes, err := r.cf.Routes() - if err != nil { - return fmt.Errorf("could not reconcile routes: %w", err) - } - - routesMap := make(map[types.NamespacedName]*routev1.Route, len(routes)) - for _, route := range routes { - routesMap[types.NamespacedName{Namespace: route.Namespace, Name: route.Name}] = route - } - - oldRoutes, err := r.client.RouteV1().Routes(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ - LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{ - MatchLabels: map[string]string{"federation.openshift.io/peer": "todo"}, - }), - }) - if err != nil { - return fmt.Errorf("failed to list routes: %w", err) - } - - oldRoutesMap := make(map[types.NamespacedName]*routev1.Route, len(oldRoutes.Items)) - for _, route := range oldRoutes.Items { - oldRoutesMap[types.NamespacedName{Namespace: route.Namespace, Name: route.Name}] = &route - } - - kind := "Route" - apiVersion := "route.openshift.io/v1" - for k, route := range routesMap { - oldRoute, ok := oldRoutesMap[k] - if !ok || !reflect.DeepEqual(&oldRoute.Spec, &route.Spec) { - // Route does not currently exist or requires an update - newRoute, err := r.client.RouteV1().Routes(route.Namespace).Apply(ctx, - &routev1apply.RouteApplyConfiguration{ - TypeMetaApplyConfiguration: v1.TypeMetaApplyConfiguration{ - Kind: &kind, - APIVersion: &apiVersion, - }, - ObjectMetaApplyConfiguration: &v1.ObjectMetaApplyConfiguration{ - Name: &route.Name, - Namespace: &route.Namespace, - Labels: route.Labels, - }, - Spec: &routev1apply.RouteSpecApplyConfiguration{ - Host: &route.Spec.Host, - To: &routev1apply.RouteTargetReferenceApplyConfiguration{ - Kind: &route.Spec.To.Kind, - Name: &route.Spec.To.Name, - }, - Port: &routev1apply.RoutePortApplyConfiguration{ - TargetPort: &route.Spec.Port.TargetPort, - }, - TLS: &routev1apply.TLSConfigApplyConfiguration{ - Termination: &route.Spec.TLS.Termination, - }, - }, - }, - metav1.ApplyOptions{ - TypeMeta: metav1.TypeMeta{ - Kind: kind, - APIVersion: apiVersion, - }, - Force: true, - FieldManager: "federation-controller", - }, - ) - if err != nil { - return fmt.Errorf("failed to apply route: %w", err) - } - log.Infof("Applied route: %v", newRoute) - } - } - - for k, oldRoute := range oldRoutesMap { - if _, ok := routesMap[k]; !ok { - err := r.client.RouteV1().Routes(oldRoute.Namespace).Delete(ctx, oldRoute.Name, metav1.DeleteOptions{}) - if client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to delete old route: %w", err) - } - log.Infof("Deleted route: %v", oldRoute) - } - } - - return nil -} diff --git a/internal/pkg/legacy/kube/service_entry_reconciler.go b/internal/pkg/legacy/kube/service_entry_reconciler.go index 47812bf3..d7aafd25 100644 --- a/internal/pkg/legacy/kube/service_entry_reconciler.go +++ b/internal/pkg/legacy/kube/service_entry_reconciler.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift-service-mesh/federation/internal/pkg/istio" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var _ Reconciler = (*ServiceEntryReconciler)(nil) diff --git a/internal/pkg/legacy/kube/workload_entry_reconciler.go b/internal/pkg/legacy/kube/workload_entry_reconciler.go index 4d45e95c..2009e3b0 100644 --- a/internal/pkg/legacy/kube/workload_entry_reconciler.go +++ b/internal/pkg/legacy/kube/workload_entry_reconciler.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift-service-mesh/federation/internal/pkg/istio" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var _ Reconciler = (*WorkloadEntryReconciler)(nil) diff --git a/internal/pkg/openshift/config_factory.go b/internal/pkg/openshift/config_factory.go deleted file mode 100644 index ea5091e3..00000000 --- a/internal/pkg/openshift/config_factory.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the License); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an AS IS BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package openshift - -import ( - "fmt" - - routev1 "github.com/openshift/api/route/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/intstr" - v1 "k8s.io/client-go/listers/core/v1" - - "github.com/openshift-service-mesh/federation/internal/pkg/config" -) - -type ConfigFactory struct { - cfg config.Federation - serviceLister v1.ServiceLister -} - -func NewConfigFactory( - cfg config.Federation, - serviceLister v1.ServiceLister, -) *ConfigFactory { - return &ConfigFactory{ - cfg: cfg, - serviceLister: serviceLister, - } -} - -func (cf *ConfigFactory) Routes() ([]*routev1.Route, error) { - createRoute := func(svcName, svcNamespace string, port int32) *routev1.Route { - return &routev1.Route{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s-%d-to-federation-ingress-gateway", svcName, svcNamespace, port), - Namespace: cf.cfg.MeshPeers.Local.ControlPlane.Namespace, - Labels: map[string]string{"federation.openshift-service-mesh.io/peer": "todo"}, - }, - Spec: routev1.RouteSpec{ - Host: fmt.Sprintf("%s-%d.%s.svc.cluster.local", svcName, port, svcNamespace), - To: routev1.RouteTargetReference{ - Kind: "Service", - Name: "federation-ingress-gateway", - }, - Port: &routev1.RoutePort{ - TargetPort: intstr.FromString(cf.cfg.MeshPeers.Local.Gateways.Ingress.Port.Name), - }, - TLS: &routev1.TLSConfig{ - Termination: routev1.TLSTerminationPassthrough, - }, - }, - } - } - - routes := []*routev1.Route{ - createRoute(fmt.Sprintf("federation-discovery-service-%s", cf.cfg.MeshPeers.Local.Name), "istio-system", 15080), - } - for _, exportLabelSelector := range cf.cfg.ExportedServiceSet.GetLabelSelectors() { - matchLabels := labels.SelectorFromSet(exportLabelSelector.MatchLabels) - services, err := cf.serviceLister.List(matchLabels) - if err != nil { - return nil, fmt.Errorf("error listing services (selector=%s): %w", matchLabels, err) - } - for _, svc := range services { - for _, port := range svc.Spec.Ports { - routes = append(routes, createRoute(svc.Name, svc.Namespace, port.Port)) - } - } - } - return routes, nil -} diff --git a/internal/pkg/legacy/xds/adsc/adsc.go b/internal/pkg/xds/adsc/adsc.go similarity index 100% rename from internal/pkg/legacy/xds/adsc/adsc.go rename to internal/pkg/xds/adsc/adsc.go diff --git a/internal/pkg/legacy/xds/adsc/response_handler.go b/internal/pkg/xds/adsc/response_handler.go similarity index 100% rename from internal/pkg/legacy/xds/adsc/response_handler.go rename to internal/pkg/xds/adsc/response_handler.go diff --git a/internal/pkg/legacy/xds/adss/adss_handler.go b/internal/pkg/xds/adss/adss_handler.go similarity index 98% rename from internal/pkg/legacy/xds/adss/adss_handler.go rename to internal/pkg/xds/adss/adss_handler.go index e43ca9a6..9f76f780 100644 --- a/internal/pkg/legacy/xds/adss/adss_handler.go +++ b/internal/pkg/xds/adss/adss_handler.go @@ -31,7 +31,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" istiolog "istio.io/istio/pkg/log" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) var log = istiolog.RegisterScope("adss", "Aggregated Discovery Service Server") diff --git a/internal/pkg/legacy/xds/adss/grpc_server.go b/internal/pkg/xds/adss/grpc_server.go similarity index 96% rename from internal/pkg/legacy/xds/adss/grpc_server.go rename to internal/pkg/xds/adss/grpc_server.go index f5d2b6fc..166378a8 100644 --- a/internal/pkg/legacy/xds/adss/grpc_server.go +++ b/internal/pkg/xds/adss/grpc_server.go @@ -22,7 +22,7 @@ import ( discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "google.golang.org/grpc" - "github.com/openshift-service-mesh/federation/internal/pkg/legacy/xds" + "github.com/openshift-service-mesh/federation/internal/pkg/xds" ) type Server struct { diff --git a/internal/pkg/legacy/xds/adss/request_handler.go b/internal/pkg/xds/adss/request_handler.go similarity index 100% rename from internal/pkg/legacy/xds/adss/request_handler.go rename to internal/pkg/xds/adss/request_handler.go diff --git a/internal/pkg/legacy/xds/push_request.go b/internal/pkg/xds/push_request.go similarity index 100% rename from internal/pkg/legacy/xds/push_request.go rename to internal/pkg/xds/push_request.go diff --git a/internal/pkg/legacy/xds/types.go b/internal/pkg/xds/types.go similarity index 50% rename from internal/pkg/legacy/xds/types.go rename to internal/pkg/xds/types.go index a1b4423c..2fb48cda 100644 --- a/internal/pkg/legacy/xds/types.go +++ b/internal/pkg/xds/types.go @@ -15,12 +15,11 @@ package xds const ( - ExportedServiceTypeUrl = "federation.openshift-service-mesh.io/v1alpha1/ExportedService" - DestinationRuleTypeUrl = "networking.istio.io/v1alpha3/DestinationRule" - GatewayTypeUrl = "networking.istio.io/v1alpha3/Gateway" - ServiceEntryTypeUrl = "networking.istio.io/v1alpha3/ServiceEntry" - WorkloadEntryTypeUrl = "networking.istio.io/v1alpha3/WorkloadEntry" - EnvoyFilterTypeUrl = "networking.istio.io/v1alpha3/EnvoyFilter" - PeerAuthenticationTypeUrl = "security.istio.io/v1beta1/PeerAuthentication" - RouteTypeUrl = "route.openshift.io/v1/Route" + ExportedServiceTypeUrl = "federation.openshift-service-mesh.io/v1alpha1/ExportedService" + DestinationRuleTypeUrl = "networking.istio.io/v1alpha3/DestinationRule" + GatewayTypeUrl = "networking.istio.io/v1alpha3/Gateway" + ServiceEntryTypeUrl = "networking.istio.io/v1alpha3/ServiceEntry" + WorkloadEntryTypeUrl = "networking.istio.io/v1alpha3/WorkloadEntry" + EnvoyFilterTypeUrl = "networking.istio.io/v1alpha3/EnvoyFilter" + RouteTypeUrl = "route.openshift.io/v1/Route" ) diff --git a/test/e2e/scenarios/remote_dns_name/main_test.go b/test/e2e/scenarios/remote_dns_name/main_test.go index b761bc4e..ca3dc727 100644 --- a/test/e2e/scenarios/remote_dns_name/main_test.go +++ b/test/e2e/scenarios/remote_dns_name/main_test.go @@ -37,6 +37,7 @@ func TestMain(m *testing.M) { Setup(setup.DeployControlPlanes("k8s")). Setup(coredns.PatchHosts). Setup(setup.InstallOrUpgradeFederationControllers(setup.RemoteAddressDNSName{})). + Setup(setup.CreateMeshFederationCR). Setup(setup.EnsureStrictMutualTLS) setup.DeployEcho(suite) diff --git a/test/e2e/scenarios/remote_ip/main_test.go b/test/e2e/scenarios/remote_ip/main_test.go index c4d350d8..3618f2d8 100644 --- a/test/e2e/scenarios/remote_ip/main_test.go +++ b/test/e2e/scenarios/remote_ip/main_test.go @@ -35,6 +35,7 @@ func TestMain(m *testing.M) { Setup(setup.CreateCACertsSecret). Setup(setup.DeployControlPlanes("k8s")). Setup(setup.InstallOrUpgradeFederationControllers()). + Setup(setup.CreateMeshFederationCR). Setup(setup.EnsureStrictMutualTLS) setup.DeployEcho(suite) diff --git a/test/e2e/scenarios/spire/main_test.go b/test/e2e/scenarios/spire/main_test.go index 7f8be597..3313c8cc 100644 --- a/test/e2e/scenarios/spire/main_test.go +++ b/test/e2e/scenarios/spire/main_test.go @@ -36,6 +36,7 @@ func TestMain(m *testing.M) { Setup(enableTrustDomainFederation). Setup(setup.DeployControlPlanes("spire")). Setup(setup.InstallOrUpgradeFederationControllers(setup.WithSpire{})). + Setup(setup.CreateMeshFederationCR). Setup(setup.EnsureStrictMutualTLS) setup.DeployEcho(suite, setup.WithSpire{}) diff --git a/test/e2e/setup/cluster_cmds.go b/test/e2e/setup/cluster_cmds.go index 8efb4d3e..8bf6abd4 100644 --- a/test/e2e/setup/cluster_cmds.go +++ b/test/e2e/setup/cluster_cmds.go @@ -73,7 +73,7 @@ func (c *Cluster) ExportService(svcName, svcNs string) error { return fmt.Errorf("failed to get service %s/%s in cluster %s: %w", svcNs, svcName, c.ContextName, err) } - svc.Labels["export-service"] = "true" + svc.Labels["export"] = "true" if _, err := c.Kube().CoreV1().Services(svcNs).Update(context.Background(), svc, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("failed to update service %s/%s in cluster %s: %w", svcNs, svcName, c.ContextName, err) } @@ -114,7 +114,6 @@ func (c *Cluster) ConfigureFederationCtrl(remoteClusters cluster.Clusters, optio "-n", "istio-system", "federation", fmt.Sprintf("%s/chart", test.ProjectRoot()), - fmt.Sprintf("--values=%s/test/testdata/federation-controller.yaml", test.ProjectRoot()), "--set", fmt.Sprintf("image.repository=%s/federation-controller", TestHub), "--set", fmt.Sprintf("image.tag=%s", TestTag), "--set", fmt.Sprintf("federation.meshPeers.local.name=%s", c.ContextName), diff --git a/test/e2e/setup/federation.go b/test/e2e/setup/federation.go index 492cbea2..712a36cb 100644 --- a/test/e2e/setup/federation.go +++ b/test/e2e/setup/federation.go @@ -25,6 +25,27 @@ import ( "istio.io/istio/pkg/test/scopes" ) +const meshFederationTemplate = ` +apiVersion: federation.openshift-service-mesh.io/v1alpha1 +kind: MeshFederation +metadata: + name: %s + namespace: istio-system +spec: + ingress: + type: istio + gateway: + selector: + app: federation-ingress-gateway + portConfig: + name: tls-passthrough + number: 15443 + export: + serviceSelectors: + matchLabels: + export: "true" +` + func InstallOrUpgradeFederationControllers(options ...CtrlOption) resource.SetupFn { return func(ctx resource.Context) error { ctx.Cleanup(func() { @@ -52,3 +73,27 @@ func InstallOrUpgradeFederationControllers(options ...CtrlOption) resource.Setup return g.Wait() } } + +func CreateMeshFederationCR(ctx resource.Context) error { + ctx.Cleanup(func() { + for _, c := range ctx.Clusters() { + localCluster := Resolve(c) + if err := c.DeleteYAMLFiles("istio-system", fmt.Sprintf(meshFederationTemplate, localCluster.ContextName)); err != nil { + scopes.Framework.Errorf("failed to delete mesh federation (cluster=%s): %v", localCluster.ContextName, err) + } + } + }) + + var g errgroup.Group + for _, c := range ctx.Clusters() { + localCluster := Resolve(c) + g.Go(func() error { + if err := c.ApplyYAMLContents("istio-system", fmt.Sprintf(meshFederationTemplate, localCluster.ContextName)); err != nil { + return fmt.Errorf("failed to apply mesh federation (cluster=%s): %w", localCluster.ContextName, err) + } + return nil + }) + } + + return g.Wait() +} diff --git a/test/testdata/federation-controller.yaml b/test/testdata/federation-controller.yaml deleted file mode 100644 index af2be81d..00000000 --- a/test/testdata/federation-controller.yaml +++ /dev/null @@ -1,13 +0,0 @@ -federation: - meshPeers: - local: - gateways: - ingress: - selector: - app: federation-ingress-gateway - exportedServiceSet: - rules: - - type: LabelSelector - labelSelectors: - - matchLabels: - export-service: "true"