feat: add support for multiple Datastores

Author: synthe102

api/v1alpha1/tenantcontrolplane_types.go

@@ -297,6 +297,14 @@ type AddonsSpec struct {
 	KubeProxy *AddonSpec `json:"kubeProxy,omitempty"`
 }
 
+// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+type DataStoreOverride struct {
+	// Resource specifies which kubernetes resource to target.
+	Resource string `json:"resource,omitempty"`
+	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+	DataStore string `json:"dataStore,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
@@ -324,8 +332,10 @@ type TenantControlPlaneSpec struct {
 	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
 	// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
-	DataStoreUsername string       `json:"dataStoreUsername,omitempty"`
-	ControlPlane      ControlPlane `json:"controlPlane"`
+	DataStoreUsername string `json:"dataStoreUsername,omitempty"`
+	// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+	DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
+	ControlPlane       ControlPlane        `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
 	// NetworkProfile specifies how the network is

internal/builders/controlplane/deployment.go

@@ -52,9 +52,15 @@ const (
 	kineInitContainerName     = "chmod"
 )
 
+type DataStoreOverrides struct {
+	Resource  string
+	DataStore kamajiv1alpha1.DataStore
+}
+
 type Deployment struct {
 	KineContainerImage string
 	DataStore          kamajiv1alpha1.DataStore
+	DataStoreOverrides []DataStoreOverrides
 	Client             client.Client
 }
 
@@ -711,6 +717,20 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
 		desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
 	}
 
+	if len(d.DataStoreOverrides) != 0 {
+		dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
+		for _, dso := range d.DataStoreOverrides {
+			httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
+
+			for _, ep := range d.DataStore.Spec.Endpoints {
+				httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
+			}
+			dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
+		}
+
+		desiredArgs["--etcd-servers-overrides"] = strings.Join(dataStoreOverridesEndpoints, ",")
+	}
+
 	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
 	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
 	return utilities.MergeMaps(current, desiredArgs, extraArgs)

internal/webhook/handlers/tcp_deployment.go

@@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje
 		}
 		t.DeploymentBuilder.DataStore = ds
 
+		dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0)
+
+		for _, dso := range tcp.Spec.DataStoreOverrides {
+			ds := kamajiv1alpha1.DataStore{}
+			if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
+				return nil, err
+			}
+			dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{
+				Resource:  dso.Resource,
+				DataStore: ds,
+			})
+		}
+		t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides
+
 		deployment := appsv1.Deployment{}
 		deployment.Name = tcp.Name
 		deployment.Namespace = tcp.Namespace