Add spec.k0s.dynamicConfig (#308)

Signed-off-by: Kimmo Lehto <[email protected]>

.github/workflows/go.yml

@@ -127,6 +127,29 @@ jobs:
       - name: Run smoke tests
         run: make smoke-files
 
+  smoke-dynamic:
+    name: Basic dynamic config smoke
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - {"name":"Go modules cache","uses":"actions/cache@v2","with":{"path":"~/go/pkg/mod\n~/.cache/go-build\n~/Library/Caches/go-build\n%LocalAppData%\\go-build\n","key":"${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}","restore-keys":"${{ runner.os }}-go-\n"}}
+      - {"name":"Compiled binary cache","uses":"actions/download-artifact@v2","with":{"name":"k0sctl","path":"."}}
+      - {"name":"Make executable","run":"chmod +x k0sctl"}
+      - {"name":"K0sctl cache","uses":"actions/cache@v2","with":{"path":"/var/cache/k0sctl\n~/.k0sctl/cache\n!*.log\n","key":"k0sctl-cache"}}
+      - {"name":"Kubectl cache","uses":"actions/cache@v2","with":{"path":"smoke-test/kubectl\n","key":"kubectl-1.21.3"}}
+      - {"name":"Go modules cache","uses":"actions/cache@v2","with":{"path":"~/go/pkg/mod\n~/.cache/go-build\n~/Library/Caches/go-build\n%LocalAppData%\\go-build\n","key":"${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}","restore-keys":"${{ runner.os }}-go-\n"}}
+      - {"name":"Docker Layer Caching For Footloose","uses":"satackey/[email protected]","continue-on-error":true}
+
+      - name: Run smoke tests
+        run: make smoke-dynamic
+
   smoke-os-override:
     name: OS override smoke test
     needs: build

Makefile

@@ -52,7 +52,7 @@ build-all: $(addprefix bin/,$(bins)) bin/checksums.md
 clean:
 	rm -rf bin/ k0sctl
 
-smoketests := smoke-basic smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore
+smoketests := smoke-basic smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic
 .PHONY: $(smoketests)
 $(smoketests): k0sctl
 	$(MAKE) -C smoke-test $@

README.md

@@ -441,6 +441,19 @@ This must be set `true` to enable the localhost connection.
 
 The version of k0s to deploy. When left out, k0sctl will default to using the latest released version of k0s or the version already running on the cluster.
 
+##### `spec.k0s.dynamicConfig` <boolean> (optional) (default: false)
+
+Enable k0s dynamic config. The setting will be automatically set to true if:
+
+* Any controller node has `--enable-dynamic-config` in `installFlags`
+* Any existing controller node has `--enable-dynamic-config` in run arguments (`k0s status -o json`)
+
+**Note:**: When running k0s in dynamic config mode, k0sctl will ONLY configure the cluster-wide configuration during the first time initialization, after that the configuration has to be managed via `k0s config edit` or `k0sctl config edit`. The node specific configuration will be updated on each apply.
+
+See also:
+
+* [k0s Dynamic Configuration](https://docs.k0sproject.io/main/dynamic-configuration/)
+
 ##### `spec.k0s.config` <mapping> (optional) (default: auto-generated)
 
 Embedded k0s cluster configuration. See [k0s configuration documentation](https://docs.k0sproject.io/main/configuration/) for details.

cmd/config_status.go

@@ -2,13 +2,11 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/k0sproject/k0sctl/analytics"
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
 	"github.com/k0sproject/rig/exec"
 
-	"github.com/mattn/go-isatty"
 	"github.com/urfave/cli/v2"
 )
 
@@ -31,10 +29,6 @@ var configStatusCommand = &cli.Command{
 	Before: actions(initLogging, startCheckUpgrade, initConfig, initAnalytics),
 	After:  actions(reportCheckUpgrade, closeAnalytics),
 	Action: func(ctx *cli.Context) error {
-		if !isatty.IsTerminal(os.Stdout.Fd()) {
-			return fmt.Errorf("output is not a terminal")
-		}
-
 		if err := analytics.Client.Publish("config-status-start", map[string]interface{}{}); err != nil {
 			return err
 		}

phase/configure_k0s.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/k0sproject/dig"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/k0sproject/rig/exec"
 	log "github.com/sirupsen/logrus"
@@ -16,28 +18,35 @@ import (
 // ConfigureK0s writes the k0s configuration to host k0s config dir
 type ConfigureK0s struct {
 	GenericPhase
+	leader *cluster.Host
 }
 
 // Title returns the phase title
 func (p *ConfigureK0s) Title() string {
 	return "Configure k0s"
 }
 
+// Prepare the phase
+func (p *ConfigureK0s) Prepare(config *v1beta1.Cluster) error {
+	p.Config = config
+	p.leader = p.Config.Spec.K0sLeader()
+	return nil
+}
+
 // Run the phase
 func (p *ConfigureK0s) Run() error {
 	if len(p.Config.Spec.K0s.Config) == 0 {
 		p.SetProp("default-config", true)
-		leader := p.Config.Spec.K0sLeader()
-		log.Warnf("%s: generating default configuration", leader)
+		log.Warnf("%s: generating default configuration", p.leader)
 
 		var cmd string
-		if leader.Exec(leader.Configurer.K0sCmdf("config create --help"), exec.Sudo(leader)) == nil {
-			cmd = leader.Configurer.K0sCmdf("config create")
+		if p.leader.Exec(p.leader.Configurer.K0sCmdf("config create --help"), exec.Sudo(p.leader)) == nil {
+			cmd = p.leader.Configurer.K0sCmdf("config create")
 		} else {
-			cmd = leader.Configurer.K0sCmdf("default-config")
+			cmd = p.leader.Configurer.K0sCmdf("default-config")
 		}
 
-		cfg, err := leader.ExecOutput(cmd, exec.Sudo(leader))
+		cfg, err := p.leader.ExecOutput(cmd, exec.Sudo(p.leader))
 		if err != nil {
 			return err
 		}
@@ -151,7 +160,13 @@ func addUnlessExist(slice *[]string, s string) {
 }
 
 func (p *ConfigureK0s) configFor(h *cluster.Host) (string, error) {
-	cfg := p.Config.Spec.K0s.Config.Dup()
+	var cfg dig.Mapping
+	// Leader will get a full config on initialize only
+	if !p.Config.Spec.K0s.DynamicConfig || (h == p.leader && h.Metadata.K0sRunningVersion == "") {
+		cfg = p.Config.Spec.K0s.Config.Dup()
+	} else {
+		cfg = p.Config.Spec.K0s.NodeConfig()
+	}
 
 	var sans []string
 

phase/gather_k0s_facts.go

@@ -126,6 +126,29 @@ func (p *GatherK0sFacts) investigateK0s(h *cluster.Host) error {
 	h.Metadata.NeedsUpgrade = p.needsUpgrade(h)
 
 	log.Infof("%s: is running k0s %s version %s", h, h.Role, h.Metadata.K0sRunningVersion)
+	if h.IsController() {
+		for _, a := range status.Args {
+			if strings.HasPrefix(a, "--enable-dynamic-config") && !strings.HasSuffix(a, "false") {
+				if !p.Config.Spec.K0s.DynamicConfig {
+					log.Warnf("%s: controller has dynamic config enabled, but spec.k0s.dynamicConfig was not set in configuration, proceeding in dynamic config mode", h)
+					p.Config.Spec.K0s.DynamicConfig = true
+				}
+			}
+		}
+		if h.InstallFlags.Include("--enable-dynamic-config") {
+			if val := h.InstallFlags.GetValue("--enable-dynamic-config"); val != "false" {
+				if !p.Config.Spec.K0s.DynamicConfig {
+					log.Warnf("%s: controller has --enable-dynamic-config in installFlags, but spec.k0s.dynamicConfig was not set in configuration, proceeding in dynamic config mode", h)
+				}
+				p.Config.Spec.K0s.DynamicConfig = true
+			}
+		}
+
+		if p.Config.Spec.K0s.DynamicConfig {
+			h.InstallFlags.AddOrReplace("--enable-dynamic-config")
+		}
+	}
+
 	if h.Metadata.NeedsUpgrade {
 		log.Warnf("%s: k0s will be upgraded", h)
 	}

phase/initialize_k0s.go

@@ -47,6 +47,12 @@ func (p *InitializeK0s) Run() error {
 	h := p.leader
 	h.Metadata.IsK0sLeader = true
 
+	if p.Config.Spec.K0s.DynamicConfig || (h.InstallFlags.Include("--enable-dynamic-config") && h.InstallFlags.GetValue("--enable-dynamic-config") != "false") {
+		p.Config.Spec.K0s.DynamicConfig = true
+		h.InstallFlags.AddOrReplace("--enable-dynamic-config")
+		p.SetProp("dynamic-config", true)
+	}
+
 	log.Infof("%s: installing k0s controller", h)
 	if err := h.Exec(h.K0sInstallCommand()); err != nil {
 		return err

pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/flags.go

@@ -13,6 +13,11 @@ func (f *Flags) Add(s string) {
 	*f = append(*f, s)
 }
 
+// Add a flag with a value
+func (f *Flags) AddWithValue(key, value string) {
+	*f = append(*f, key+" "+value)
+}
+
 // AddUnlessExist adds a flag unless one with the same prefix exists
 func (f *Flags) AddUnlessExist(s string) {
 	if f.Include(s) {

pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Masterminds/semver"
 	"github.com/alessio/shellescape"
 	"github.com/avast/retry-go"
 	"github.com/creasty/defaults"
@@ -24,9 +25,10 @@ const K0sMinVersion = "0.11.0-rc1"
 
 // K0s holds configuration for bootstraping a k0s cluster
 type K0s struct {
-	Version  string      `yaml:"version"`
-	Config   dig.Mapping `yaml:"config,omitempty"`
-	Metadata K0sMetadata `yaml:"-"`
+	Version       string      `yaml:"version"`
+	DynamicConfig bool        `yaml:"dynamicConfig"`
+	Config        dig.Mapping `yaml:"config,omitempty"`
+	Metadata      K0sMetadata `yaml:"-"`
 }
 
 // K0sMetadata contains gathered information about k0s cluster
@@ -47,6 +49,8 @@ func (k *K0s) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return defaults.Set(k)
 }
 
+const k0sDynamicSince = "1.22.2+k0s.2"
+
 func validateVersion(value interface{}) error {
 	vs, ok := value.(string)
 	if !ok {
@@ -74,9 +78,31 @@ func (k *K0s) Validate() error {
 	return validation.ValidateStruct(k,
 		validation.Field(&k.Version, validation.Required),
 		validation.Field(&k.Version, validation.By(validateVersion)),
+		validation.Field(&k.DynamicConfig, validation.By(k.validateMinDynamic())),
 	)
 }
 
+func (k *K0s) validateMinDynamic() func(interface{}) error {
+	return func(value interface{}) error {
+		dc, ok := value.(bool)
+		if !ok {
+			return fmt.Errorf("not a boolean")
+		}
+		if !dc {
+			return nil
+		}
+		v, err := semver.NewVersion(k.Version)
+		if err != nil {
+			return fmt.Errorf("failed to parse k0s version: %w", err)
+		}
+		dynamicSince, _ := semver.NewVersion(k0sDynamicSince)
+		if v.LessThan(dynamicSince) {
+			return fmt.Errorf("dynamic config only available since k0s version %s", k0sDynamicSince)
+		}
+		return nil
+	}
+}
+
 // SetDefaults (implements defaults Setter interface) defaults the version to latest k0s version
 func (k *K0s) SetDefaults() {
 	latest, err := version.LatestReleaseByPrerelease(k0sctl.IsPre() || k0sctl.Version == "0.0.0")
@@ -88,6 +114,20 @@ func (k *K0s) SetDefaults() {
 	k.Version = strings.TrimPrefix(k.Version, "v")
 }
 
+func (k *K0s) NodeConfig() dig.Mapping {
+	return dig.Mapping{
+		"apiVersion": k.Config.DigString("apiVersion"),
+		"kind":       k.Config.DigString("kind"),
+		"Metadata": dig.Mapping{
+			"name": k.Config.DigMapping("metadata")["name"],
+		},
+		"spec": dig.Mapping{
+			"api":     k.Config.DigMapping("spec", "api"),
+			"storage": k.Config.DigMapping("spec", "storage"),
+		},
+	}
+}
+
 // GenerateToken runs the k0s token create command
 func (k K0s) GenerateToken(h *Host, role string, expiry time.Duration) (string, error) {
 	var k0sFlags Flags

smoke-test/Makefile

@@ -22,6 +22,9 @@ id_rsa_k0s:
 smoke-basic: $(footloose) id_rsa_k0s k0sctl
 	./smoke-basic.sh
 
+smoke-dynamic: $(footloose) id_rsa_k0s k0sctl
+	./smoke-dynamic.sh
+
 smoke-files: $(footloose) id_rsa_k0s k0sctl
 	./smoke-files.sh
 

smoke-test/k0sctl-dynamic.yaml

@@ -0,0 +1,24 @@
+apiVersion: k0sctl.k0sproject.io/v1beta1
+kind: cluster
+spec:
+  hosts:
+    - role: controller
+      uploadBinary: true
+      ssh:
+        address: "127.0.0.1"
+        port: 9022
+        keyPath: ./id_rsa_k0s
+    - role: worker
+      uploadBinary: true
+      os: "$OS_OVERRIDE"
+      ssh:
+        address: "127.0.0.1"
+        port: 9023
+        keyPath: ./id_rsa_k0s
+  k0s:
+    version: "${K0S_VERSION}"
+    dynamicConfig: true
+    config:
+      spec:
+        telemetry:
+          enabled: false

smoke-test/smoke-dynamic.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+K0SCTL_CONFIG=${K0SCTL_CONFIG:-"k0sctl-dynamic.yaml"}
+
+set -e
+
+
+. ./smoke.common.sh
+trap cleanup EXIT
+
+deleteCluster
+createCluster
+
+echo "* Starting apply"
+../k0sctl apply --config "${K0SCTL_CONFIG}" --debug
+echo "* Apply OK"
+
+max_retry=5
+counter=0
+echo "* Verifying dynamic config reconciliation was a success"
+until ../k0sctl config status -o json --config "${K0SCTL_CONFIG}" | grep -q "SuccessfulReconcile"
+do
+   [[ counter -eq $max_retry ]] && echo "Failed!" && exit 1
+   echo "* Waiting for a couple of seconds to retry"
+   sleep 5
+   ((counter++))
+done
+
+echo "* OK"
+
+echo "* Dynamic config reconciliation status:"
+../k0sctl config status --config "${K0SCTL_CONFIG}"
+
+echo "* Done"

smoke-test/smoke.common.sh

@@ -3,7 +3,7 @@ FOOTLOOSE_TEMPLATE=${FOOTLOOSE_TEMPLATE:-"footloose.yaml.tpl"}
 export LINUX_IMAGE=${LINUX_IMAGE:-"quay.io/footloose/ubuntu18.04"}
 export PRESERVE_CLUSTER=${PRESERVE_CLUSTER:-""}
 export DISABLE_TELEMETRY=true
-export K0S_VERSION=${K0S_VERSION:-"1.21.1+k0s.0"}
+export K0S_VERSION
 
 function createCluster() {
   envsubst < "${FOOTLOOSE_TEMPLATE}" > footloose.yaml