Merge pull request #101 from projectsyn/add/k8s-migration

Migrate deprecated k3s to vanilla k8s

Author: Kidswiss

class/defaults.yml

@@ -9,17 +9,17 @@ parameters:
     charts:
       vcluster:
         source: https://charts.loft.sh
-        version: 0.24.1
+        version: 0.29.1
 
     images:
-      k3s:
-        registry: docker.io
-        image: rancher/k3s
-        tag: v1.32.9-k3s1
+      k8s:
+        registry: ghcr.io
+        image: loft-sh/kubernetes
+        tag: v1.32.9
       syncer:
         registry: ghcr.io
         image: loft-sh/vcluster-oss
-        tag: "0.24.1"
+        tag: "0.29.1"
       oc:
         registry: quay.io
         image: appuio/oc
@@ -30,8 +30,10 @@ parameters:
       size: 5Gi
       class_name: null
 
-    k3s:
-      additional_args: []
+    k8s:
+      additional_apiserver_args: []
+      additional_controllermanager_args: []
+      additional_scheduler_args: []
 
     ingress:
       enabled: true
@@ -41,10 +43,10 @@ parameters:
 
     backing_store: {}
 
-    additional_manifests: nil
+    additional_manifests: ""
 
     resources:
-      k3s:
+      k8s:
         requests:
           cpu: 40m
           memory: 64Mi
@@ -66,22 +68,28 @@ parameters:
       controlPlane:
         distro:
           k3s:
+            enabled: false
+          k8s:
             enabled: true
-            extraArgs: ${vcluster:k3s:additional_args}
+            apiServer:
+              extraArgs: ${vcluster:k8s:additional_apiserver_args}
+            controllerManager:
+              extraArgs: ${vcluster:k8s:additional_controllermanager_args}
+            scheduler:
+              extraArgs: ${vcluster:k8s:additional_scheduler_args}
             image:
-              registry: ${vcluster:images:k3s:registry}
-              repository: ${vcluster:images:k3s:image}
-              tag: ${vcluster:images:k3s:tag}
-            resources: ${vcluster:resources:k3s}
+              registry: ${vcluster:images:k8s:registry}
+              repository: ${vcluster:images:k8s:image}
+              tag: ${vcluster:images:k8s:tag}
+            resources: ${vcluster:resources:k8s}
 
         statefulSet:
           security:
             containerSecurityContext:
               allowPrivilegeEscalation: false
               runAsUser: null
               runAsGroup: null
-          resources:
-            ${vcluster:resources:syncer}
+          resources: ${vcluster:resources:syncer}
           image:
             registry: ${vcluster:images:syncer:registry}
             repository: ${vcluster:images:syncer:image}

component/scripts/patch-route.sh

@@ -15,7 +15,7 @@ echo "Check if route is already patched"
 
 patched=$(kubectl -n "$NAMESPACE" get route "$route_name" -o jsonpath='{.spec.tls.destinationCACertificate}')
 
-if [ "$patched" != "" ]; then
+if [ "$patched" = "$(cat $cert)" ]; then
     echo "Route is already patched. Nothing to do"
     exit
 fi

component/scripts/synthesize.sh

@@ -6,7 +6,7 @@ vcluster_kubeconfig=./config
 
 echo "Setting server URL..."
 
-kubectl --kubeconfig "$vcluster_kubeconfig" config set clusters.local.server "$VCLUSTER_SERVER_URL"
+kubectl --kubeconfig "$vcluster_kubeconfig" config set clusters.kubernetes.server "$VCLUSTER_SERVER_URL"
 
 echo "Checking for namespace 'syn'..."
 

docs/modules/ROOT/pages/how-tos/upgrade-2.x-to-3.x.adoc

@@ -0,0 +1,63 @@
+= Upgrade from 2.x to 3.x
+
+Version 3.x removes the deprecated k3s distro from the configuration.
+It is replaced with a vanilla k8s distribution.
+The helmchart and vcluster version also get bumped to 0.29.1 to support the new configurations.
+
+[WARNING]
+====
+Before upgrading to version 3.x, please ensure you have a backup of the vcluster (backingStore PVC).
+
+Migrating from k3s to k8s is automatic and should not need any manual steps.
+However if there are issues you can consult https://www.vcluster.com/docs/vcluster/deploy/upgrade/distro-migration#troubleshoot-migration-issues[the official migration docs].
+====
+
+== Migrate remove fields
+
+All fields named `k3s` have been renamed to `k8s`.
+But unfortunately the configuration is not a drop-in replacement.
+
+The k3s image was an all-in-one kind of deal.
+There was only a single `additional_args` parameter that got passed to k3s and those parameters specified if they are for the apiServer or any other component of k8s.
+However with the vanilla k8s distribution it's now split across three differenct helm values.
+Each with their own `additional_args` setting:
+
+- apiServer: `additional_apiserver_args`
+- controllerManager: `additional_controlmanager_args`
+- scheduler: `additional_scheduler_args`
+
+[WARNING]
+====
+The migration supports sqlite and embedded etcd backingStores.
+However the embedded etcd backingStores are an enterprise feature.
+OSS versions of vcluster with etcd will have a seperately deployed etcd.
+Migrations still work, but the certificates will get re-created.
+
+After a migration with etcd kubeconfigs and certificates need to be updated wherever they are referenced.
+====
+
+=== Migrating a vcluster with apiServer settings
+
+[source,diff]
+----
+parameters:
+  vcluster:
+-    k3s:
++    k8s:
+      additional_args:
+-        - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local
+-        - --kube-apiserver-arg=oidc-client-id=local
+-        - --kube-apiserver-arg=oidc-username-claim=email
+-        - --kube-apiserver-arg=oidc-groups-claim=groups
++        - --oidc-issuer-url=https://id.local/auth/realms/local
++        - --oidc-client-id=local
++        - --oidc-username-claim=email
++        - --oidc-groups-claim=groups
+----
+
+[NOTE]
+====
+The output of `kubectl get nodes` will still show nodes with `v1.32.9+k3s1`.
+That's expected.
+To check th actual used image use: `kubectl -n $vlcusterns get pods $vclusterpod -oyaml | grep image:`.
+====

docs/modules/ROOT/pages/references/parameters.adoc

@@ -41,7 +41,7 @@ Dictionary containing the container images used by this component.
 
 The `kubectl` image is used to create OCP routes. The `kubectl` and `jq` binaries are required in this image.
 
-== `k3s.additional_args`
+== `k8s.additional_apiserver_args`
 
 [horizontal]
 type:: list
@@ -50,16 +50,47 @@ example::
 +
 [source,yaml]
 ----
-k3s:
-  additional_args:
-  - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local
-  - --kube-apiserver-arg=oidc-client-id=local
-  - --kube-apiserver-arg=oidc-username-claim=email
-  - --kube-apiserver-arg=oidc-groups-claim=groups
+k8s:
+  additional_apiserver_args:
+  - --oidc-issuer-url=https://id.local/auth/realms/local
+  - --oidc-client-id=local
+  - --oidc-username-claim=email
+  - --oidc-groups-claim=groups
 ----
 
-Additional arguments for the k3s cluster.
+Additional arguments for the k8s apiserver.
 
+== `k8s.additional_controllermanager_args`
+
+[horizontal]
+type:: list
+default:: `[]`
+example::
++
+[source,yaml]
+----
+k8s:
+  additional_controllermanager_args:
+  - --arg1=foobar
+----
+
+Additional arguments for the k8s controllerManager.
+
+== `k8s.additional_scheduler_args`
+
+[horizontal]
+type:: list
+default:: `[]`
+example::
++
+[source,yaml]
+----
+k8s:
+  additional_scheduler_args:
+  - --arg1=foobar
+----
+
+Additional arguments for the k8s scheduler.
 
 == `backing_store`
 
@@ -77,7 +108,7 @@ default::
 [source,yaml]
 ----
 resources:
-  k3s:
+  k8s:
     requests:
       cpu: 40m
       memory: 64Mi
@@ -92,7 +123,7 @@ resources:
       memory: 2Gi
 ----
 
-The resource requests and limits for the k3s and syncer containers.
+The resource requests and limits for the k8s and syncer containers.
 
 
 == `ingress.host`
@@ -206,7 +237,7 @@ Manifests that should be applied to the vcluster after startup.
 [horizontal]
 type:: dict
 
-You can override the default helm values here. The default configuration comes with k3s as well as ingress enabled. See defaults.yaml for the exact configuration.
+You can override the default helm values here. The default configuration comes with k8s as well as ingress enabled. See defaults.yaml for the exact configuration.
 
 
 == Example
@@ -215,12 +246,12 @@ You can override the default helm values here. The default configuration comes w
 ----
 ingress:
   host: testcluster.local
-k3s:
-  additional_args:
-    - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local
-    - --kube-apiserver-arg=oidc-client-id=local
-    - --kube-apiserver-arg=oidc-username-claim=email
-    - --kube-apiserver-arg=oidc-groups-claim=groups
+k8s:
+  additional_apiserver_args:
+    - --oidc-issuer-url=https://id.local/auth/realms/local
+    - --oidc-client-id=local
+    - --oidc-username-claim=email
+    - --oidc-groups-claim=groups
 backing_store:
   etcd:
     deploy:

docs/modules/ROOT/partials/nav.adoc

@@ -2,6 +2,7 @@
 
 .How-to guides
 * xref:how-tos/upgrade-1.x-to-2.x.adoc[Upgrade 1.x to 2.x]
+* xref:how-tos/upgrade-2.x-to-3.x.adoc[Upgrade 2.x to 3.x]
 * xref:tutorials/installation-openshift.adoc[Installation On OpenShift]
 * xref:how-tos/oidc.adoc[Setup OIDC Integration]