[WIP] feat: add support for multiple Datastores

api/v1alpha1/tenantcontrolplane_types.go

@@ -297,6 +297,14 @@ type AddonsSpec struct {
 	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
 }
 
+// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+type DataStoreOverride struct {
+	// Resource specifies which kubernetes resource to target.
+	Resource string `json:"resource,omitempty"`
+	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+	DataStore string `json:"dataStore,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
@@ -324,8 +332,10 @@ type TenantControlPlaneSpec struct {
 	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
 	// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
-	DataStoreUsername string       `json:"dataStoreUsername,omitempty"`
-	ControlPlane      ControlPlane `json:"controlPlane"`
+	DataStoreUsername string `json:"dataStoreUsername,omitempty"`
+	// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+	DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
+	ControlPlane       ControlPlane        `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
 	// NetworkProfile specifies how the network is

charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml

@@ -6760,6 +6760,19 @@ versions:
                   Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
                   Migration from one DataStore to another backed by a different Driver is not supported.
                 type: string
+              dataStoreOverrides:
+                description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+                items:
+                  description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+                  properties:
+                    dataStore:
+                      description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+                      type: string
+                    resource:
+                      description: Resource specifies which kubernetes resource to target.
+                      type: string
+                  type: object
+                type: array
               dataStoreSchema:
                 description: |-
                   DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -6768,6 +6768,19 @@ spec:
                     Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
                     Migration from one DataStore to another backed by a different Driver is not supported.
                   type: string
+                dataStoreOverrides:
+                  description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+                  items:
+                    description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+                    properties:
+                      dataStore:
+                        description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+                        type: string
+                      resource:
+                        description: Resource specifies which kubernetes resource to target.
+                        type: string
+                    type: object
+                  type: array
                 dataStoreSchema:
                   description: |-
                     DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

internal/builders/controlplane/controlplane_suite_test.go

@@ -0,0 +1,16 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplane_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestControlpalne(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ControlPlane Suite")
+}

internal/builders/controlplane/deployment.go

@@ -52,9 +52,15 @@ const (
 	kineInitContainerName     = "chmod"
 )
 
+type DataStoreOverrides struct {
+	Resource  string
+	DataStore kamajiv1alpha1.DataStore
+}
+
 type Deployment struct {
 	KineContainerImage string
 	DataStore          kamajiv1alpha1.DataStore
+	DataStoreOverrides []DataStoreOverrides
 	Client             client.Client
 }
 
@@ -711,11 +717,28 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
 		desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
 	}
 
+	if len(d.DataStoreOverrides) != 0 {
+		desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides()
+	}
+
 	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
 	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
 	return utilities.MergeMaps(current, desiredArgs, extraArgs)
 }
 
+func (d Deployment) etcdServersOverrides() string {
+	dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
+	for _, dso := range d.DataStoreOverrides {
+		httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
+
+		for _, ep := range dso.DataStore.Spec.Endpoints {
+			httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
+		}
+		dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
+	}
+	return strings.Join(dataStoreOverridesEndpoints, ",")
+}
+
 func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
 	return &corev1.SecretProjection{
 		LocalObjectReference: corev1.LocalObjectReference{

internal/builders/controlplane/deployment_test.go

@@ -0,0 +1,48 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplane
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Controlplane Deployment", func() {
+	var (
+		d Deployment
+	)
+	BeforeEach(func() {
+		d = Deployment{
+			DataStoreOverrides: []DataStoreOverrides{{
+				Resource: "/events",
+				DataStore: kamajiv1alpha1.DataStore{
+					Spec: kamajiv1alpha1.DataStoreSpec{
+						Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"},
+					},
+				},
+			}},
+		}
+	})
+
+	Describe("DataStoreOverrides flag generation", func() {
+		It("should generate valid --etcd-servers-overrides value", func() {
+			etcdSerVersOverrides := d.etcdServersOverrides()
+			Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2"))
+		})
+		It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() {
+			d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{
+				Resource: "/pods",
+				DataStore: kamajiv1alpha1.DataStore{
+					Spec: kamajiv1alpha1.DataStoreSpec{
+						Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"},
+					},
+				},
+			})
+			etcdSerVersOverrides := d.etcdServersOverrides()
+			Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
+		})
+	})
+})

internal/webhook/handlers/tcp_datastore.go

@@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe
 			return nil, t.check(ctx, tcp.Spec.DataStore)
 		}
 
-		return nil, nil
+		return nil, t.checkDataStoreOverrides(ctx, tcp)
 	}
 }
 
@@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st
 
 	return nil
 }
+
+func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error {
+	overrideCheck := make(map[string]struct{}, 0)
+	for _, ds := range tcp.Spec.DataStoreOverrides {
+		if _, exists := overrideCheck[ds.Resource]; !exists {
+			overrideCheck[ds.Resource] = struct{}{}
+		} else {
+			return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides")
+		}
+		if err := t.check(ctx, ds.DataStore); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/webhook/handlers/tcp_datastore_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package handlers
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("TCP Datastore webhook", func() {
+	var (
+		ctx context.Context
+		t   TenantControlPlaneDataStore
+		tcp *kamajiv1alpha1.TenantControlPlane
+	)
+	BeforeEach(func() {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
+
+		ctx = context.Background()
+		t = TenantControlPlaneDataStore{
+			Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+			}, &kamajiv1alpha1.DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bar",
+				},
+			}).Build(),
+		}
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "tcp",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{},
+		}
+	})
+	Describe("validation should succeed without DataStoreOverrides", func() {
+		It("should validate TCP without DataStoreOverrides", func() {
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Describe("validation should fail with duplicate resources in DataStoreOverrides", func() {
+		It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/event",
+				DataStore: "foo",
+			}, {
+				Resource:  "/event",
+				DataStore: "bar",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("validation should succeed with valid DataStoreOverrides", func() {
+		It("should validate TCP with valid DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/leases",
+				DataStore: "foo",
+			}, {
+				Resource:  "/event",
+				DataStore: "bar",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Describe("validation should fail with nonexistent DataStoreOverrides", func() {
+		It("should fail to validate TCP with nonexistent DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/leases",
+				DataStore: "baz",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

internal/webhook/handlers/tcp_deployment.go

@@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje
 		}
 		t.DeploymentBuilder.DataStore = ds
 
+		dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0, len(tcp.Spec.DataStoreOverrides))
+
+		for _, dso := range tcp.Spec.DataStoreOverrides {
+			ds := kamajiv1alpha1.DataStore{}
+			if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
+				return nil, err
+			}
+			dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{
+				Resource:  dso.Resource,
+				DataStore: ds,
+			})
+		}
+		t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides
+
 		deployment := appsv1.Deployment{}
 		deployment.Name = tcp.Name
 		deployment.Namespace = tcp.Namespace