Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add workload cluster kubeconfig certs rotation #859

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add workload cluster kubeconfig certs rotation #859

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 44 additions & 6 deletions internal/controller/controlplane/k0s_controlplane_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
kubeadmbootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
capiutil "sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/annotations"
"sigs.k8s.io/cluster-api/util/certs"
"sigs.k8s.io/cluster-api/util/collections"
"sigs.k8s.io/cluster-api/util/failuredomains"
"sigs.k8s.io/cluster-api/util/kubeconfig"
Expand Down Expand Up @@ -201,26 +202,58 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
}

func (c *K0sController) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) error {
logger := log.FromContext(ctx, "cluster", cluster.Name, "kcp", kcp.Name)

if cluster.Spec.ControlPlaneEndpoint.IsZero() {
return fmt.Errorf("control plane endpoint is not set: %w", ErrNotReady)
}

secretName := secret.Name(cluster.Name, secret.Kubeconfig)
err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, &corev1.Secret{})
kubeconfigSecrets := []*corev1.Secret{}

// Always rotate certificates if needed.
defer func() {
for _, kc := range kubeconfigSecrets {
needsRotation, err := kubeconfig.NeedsClientCertRotation(kc, certs.ClientCertificateRenewalDuration)
if err != nil {
logger.Error(err, "Failed to check if certificate needs rotation.")
return
}

if needsRotation {
logger.Info("Rotating kubeconfig secret", "Secret", kc.GetName())
if err := c.regenerateKubeconfigSecret(ctx, kc); err != nil {
logger.Error(err, "Failed to regenerate kubeconfig")
return
}
}
}
}()

workloadClusterKubeconfigSecret, err := secret.GetFromNamespacedName(ctx, c.Client, capiutil.ObjectKey(cluster), secret.Kubeconfig)
if err != nil {
if apierrors.IsNotFound(err) {
return kubeconfig.CreateSecret(ctx, c.Client, cluster)
}

return err
}
kubeconfigSecrets = append(kubeconfigSecrets, workloadClusterKubeconfigSecret)

if kcp.Spec.K0sConfigSpec.Tunneling.Enabled {
clusterKey := client.ObjectKey{
Name: cluster.GetName(),
Namespace: cluster.GetNamespace(),
}

if kcp.Spec.K0sConfigSpec.Tunneling.Mode == "proxy" {

secretName := secret.Name(cluster.Name+"-proxied", secret.Kubeconfig)
err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, &corev1.Secret{})

proxiedKubeconfig := &corev1.Secret{}
err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, proxiedKubeconfig)
if err != nil {
if apierrors.IsNotFound(err) {
kc, err := c.generateKubeconfig(ctx, cluster, fmt.Sprintf("https://%s", cluster.Spec.ControlPlaneEndpoint.String()))
kc, err := c.generateKubeconfig(ctx, clusterKey, fmt.Sprintf("https://%s", cluster.Spec.ControlPlaneEndpoint.String()))
if err != nil {
return err
}
Expand All @@ -236,12 +269,16 @@ func (c *K0sController) reconcileKubeconfig(ctx context.Context, cluster *cluste
}
return err
}
kubeconfigSecrets = append(kubeconfigSecrets, proxiedKubeconfig)

} else {
secretName := secret.Name(cluster.Name+"-tunneled", secret.Kubeconfig)
err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, &corev1.Secret{})

tunneledKubeconfig := &corev1.Secret{}
err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, tunneledKubeconfig)
if err != nil {
if apierrors.IsNotFound(err) {
kc, err := c.generateKubeconfig(ctx, cluster, fmt.Sprintf("https://%s:%d", kcp.Spec.K0sConfigSpec.Tunneling.ServerAddress, kcp.Spec.K0sConfigSpec.Tunneling.TunnelingNodePort))
kc, err := c.generateKubeconfig(ctx, clusterKey, fmt.Sprintf("https://%s:%d", kcp.Spec.K0sConfigSpec.Tunneling.ServerAddress, kcp.Spec.K0sConfigSpec.Tunneling.TunnelingNodePort))
if err != nil {
return err
}
Expand All @@ -253,6 +290,7 @@ func (c *K0sController) reconcileKubeconfig(ctx context.Context, cluster *cluste
}
return err
}
kubeconfigSecrets = append(kubeconfigSecrets, tunneledKubeconfig)
}
}

Expand Down
51 changes: 47 additions & 4 deletions internal/controller/controlplane/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"

"github.com/imdario/mergo"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -39,9 +40,8 @@ func (c *K0sController) getMachineTemplate(ctx context.Context, kcp *cpv1beta1.K
return machineTemplate, nil
}

func (c *K0sController) generateKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, endpoint string) (*api.Config, error) {
clusterName := util.ObjectKey(cluster)
clusterCA, err := secret.GetFromNamespacedName(ctx, c.Client, clusterName, secret.ClusterCA)
func (c *K0sController) generateKubeconfig(ctx context.Context, clusterKey client.ObjectKey, endpoint string) (*api.Config, error) {
clusterCA, err := secret.GetFromNamespacedName(ctx, c.Client, clusterKey, secret.ClusterCA)
if err != nil {
if apierrors.IsNotFound(err) {
return nil, kubeconfig.ErrDependentCertificateNotFound
Expand All @@ -63,7 +63,7 @@ func (c *K0sController) generateKubeconfig(ctx context.Context, cluster *cluster
return nil, fmt.Errorf("CA private key not found: %w", err)
}

cfg, err := kubeconfig.New(clusterName.Name, endpoint, cert, key)
cfg, err := kubeconfig.New(clusterKey.Name, endpoint, cert, key)
if err != nil {
return nil, fmt.Errorf("failed to generate a kubeconfig: %w", err)
}
Expand Down Expand Up @@ -92,6 +92,49 @@ func (c *K0sController) createKubeconfigSecret(ctx context.Context, cfg *api.Con
return c.Create(ctx, kcSecret)
}

func (c *K0sController) regenerateKubeconfigSecret(ctx context.Context, kubeconfigSecret *v1.Secret) error {
clusterName, _, err := secret.ParseSecretName(kubeconfigSecret.Name)
if err != nil {
return fmt.Errorf("failed to parse secret name: %w", err)
}
data, ok := kubeconfigSecret.Data[secret.KubeconfigDataName]
if !ok {
return fmt.Errorf("missing key %q in secret data: %w", secret.KubeconfigDataName, err)
}

oldConfig, err := clientcmd.Load(data)
if err != nil {
return fmt.Errorf("failed to convert kubeconfig Secret into a clientcmdapi.Config: %w", err)
}

endpoint := oldConfig.Clusters[clusterName].Server

clusterKey := client.ObjectKey{
Name: clusterName,
// The namespace of the current kubeconfig secret can be used, as it is always
// created in the cluster's namespace.
Namespace: kubeconfigSecret.Namespace,
}
newConfig, err := c.generateKubeconfig(ctx, clusterKey, endpoint)
if err != nil {
return err
}

// The proxy URL needs to be set for the new secret using the old value. That way we
// cover cases when tunneling mode = "proxy" and proxy url exists.
for cn := range newConfig.Clusters {
newConfig.Clusters[cn].ProxyURL = oldConfig.Clusters[clusterName].ProxyURL
}

out, err := clientcmd.Write(*newConfig)
if err != nil {
return fmt.Errorf("failed to serialize config to yaml: %w", err)
}
kubeconfigSecret.Data[secret.KubeconfigDataName] = out

return c.Update(ctx, kubeconfigSecret)
}

func (c *K0sController) getKubeClient(ctx context.Context, cluster *clusterv1.Cluster) (*kubernetes.Clientset, error) {
return k0smoutil.GetKubeClient(ctx, c.Client, cluster)
}
Expand Down
Loading