From 35e3a3a73b9af63c5546ea13246bc71b460cdab9 Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Wed, 18 Oct 2023 16:11:52 +0100 Subject: [PATCH] Add plan and node-token commands * plan can be used to generate a bash script to install a HA cluster - a minimum set of options are included, PRs are welcome to add anything else you may need * node-token obtains a node token for k3sup join, which is required when creating large clusters via automation, so that there are not too many SSH connections created when it's not required * a helper method was added to obtain an SSH connection, rather than repeating it in each command Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- .gitignore | 3 + cmd/install.go | 120 +++++++++++++++++++++-------------- cmd/join.go | 112 +++++++++++++-------------------- cmd/node-token.go | 155 ++++++++++++++++++++++++++++++++++++++++++++++ cmd/plan.go | 152 +++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- main.go | 4 ++ 7 files changed, 433 insertions(+), 115 deletions(-) create mode 100644 cmd/node-token.go create mode 100644 cmd/plan.go diff --git a/.gitignore b/.gitignore index 5437d701..47af3e10 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ kubeconfig .idea/ mc /install.sh +/*.json +*.txt +/bootstrap.sh diff --git a/cmd/install.go b/cmd/install.go index 5798482d..b9013ae2 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) var kubeconfig []byte @@ -117,6 +117,7 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som command.Flags().String("tls-san", "", "Use an additional IP or hostname for the API server") command.PreRunE = func(command *cobra.Command, args []string) error { + local, err := command.Flags().GetBool("local") if err != nil { return err @@ -286,62 +287,25 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som return nil } - port, _ := command.Flags().GetInt("ssh-port") - fmt.Println("Public IP: " + host) + port, _ := command.Flags().GetInt("ssh-port") user, _ := command.Flags().GetString("user") sshKey, _ := command.Flags().GetString("ssh-key") sshKeyPath := expandPath(sshKey) address := fmt.Sprintf("%s:%d", host, port) - var sshOperator *operator.SSHOperator - var initialSSHErr error - if runtime.GOOS != "windows" { - - var sshAgentAuthMethod ssh.AuthMethod - sshAgentAuthMethod, initialSSHErr = sshAgentOnly() - if initialSSHErr == nil { - // Try SSH agent without parsing key files, will succeed if the user - // has already added a key to the SSH Agent, or if using a configured - // smartcard - config := &ssh.ClientConfig{ - User: user, - Auth: []ssh.AuthMethod{sshAgentAuthMethod}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - sshOperator, initialSSHErr = operator.NewSSHOperator(address, config) - } - } else { - initialSSHErr = errors.New("ssh-agent unsupported on windows") + sshOperator, sshOperatorDone, errored, err := connectOperator(user, address, sshKeyPath) + if errored { + return err } - // If the initial connection attempt fails fall through to the using - // the supplied/default private key file - if initialSSHErr != nil { - publicKeyFileAuth, closeSSHAgent, err := loadPublickey(sshKeyPath) - if err != nil { - return fmt.Errorf("unable to load the ssh key with path %q: %w", sshKeyPath, err) - } - - defer closeSSHAgent() + if sshOperatorDone != nil { - config := &ssh.ClientConfig{ - User: user, - Auth: []ssh.AuthMethod{publicKeyFileAuth}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - sshOperator, err = operator.NewSSHOperator(address, config) - if err != nil { - return fmt.Errorf("unable to connect to %s over ssh: %w", address, err) - } + defer sshOperatorDone() } - defer sshOperator.Close() - if !skipInstall { if printCommand { @@ -360,6 +324,7 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som if printCommand { fmt.Printf("ssh: %s\n", getConfigcommand) } + if err = obtainKubeconfig(sshOperator, getConfigcommand, host, context, localKubeconfig, merge, printConfig); err != nil { return err } @@ -370,6 +335,71 @@ Provide the --local-path flag with --merge if a kubeconfig already exists in som return command } +type DoneFunc func() + +// connectOperator +// +// Try SSH agent without parsing key files, will succeed if the user +// has already added a key to the SSH Agent, or if using a configured +// smartcard. +// +// If the initial connection attempt fails fall through to the using +// the supplied/default private key file +// DoneFunc should be called by the caller to close the SSH connection when done +func connectOperator(user string, address string, sshKeyPath string) (*operator.SSHOperator, DoneFunc, bool, error) { + var sshOperator *operator.SSHOperator + var initialSSHErr error + var closeSSHAgentFunc func() error + + doneFunc := func() { + if sshOperator != nil { + sshOperator.Close() + } + if closeSSHAgentFunc != nil { + closeSSHAgentFunc() + } + } + + if runtime.GOOS != "windows" { + var sshAgentAuthMethod ssh.AuthMethod + sshAgentAuthMethod, initialSSHErr = sshAgentOnly() + if initialSSHErr == nil { + + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{sshAgentAuthMethod}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + sshOperator, initialSSHErr = operator.NewSSHOperator(address, config) + } + } else { + initialSSHErr = errors.New("ssh-agent unsupported on windows") + } + + if initialSSHErr != nil { + publicKeyFileAuth, closeSSHAgent, err := loadPublickey(sshKeyPath) + if err != nil { + return nil, nil, true, fmt.Errorf("unable to load the ssh key with path %q: %w", sshKeyPath, err) + } + + defer closeSSHAgent() + + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{publicKeyFileAuth}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + sshOperator, err = operator.NewSSHOperator(address, config) + if err != nil { + return nil, nil, true, fmt.Errorf("unable to connect to %s over ssh: %w", address, err) + } + } + + return sshOperator, doneFunc, false, nil +} + func sshAgentOnly() (ssh.AuthMethod, error) { sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")) if err != nil { @@ -540,7 +570,7 @@ func loadPublickey(path string) (ssh.AuthMethod, func() error, error) { fmt.Printf("Enter passphrase for '%s': ", path) STDIN := int(os.Stdin.Fd()) - bytePassword, _ := terminal.ReadPassword(STDIN) + bytePassword, _ := term.ReadPassword(STDIN) // Ignore any error from reading stdin to retain existing behaviour for unit test in // install_test.go diff --git a/cmd/join.go b/cmd/join.go index c8720895..c5f23942 100644 --- a/cmd/join.go +++ b/cmd/join.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "net" + "os" "path" "runtime" "strings" @@ -65,6 +66,7 @@ func MakeJoin() *cobra.Command { command.Flags().Bool("server", false, "Join the cluster as a server rather than as an agent for the embedded etcd mode") command.Flags().Bool("print-command", false, "Print a command that you can use with SSH to manually recover from an error") + command.Flags().String("node-token-path", "", "prefetched token used by nodes to join the cluster") command.Flags().String("k3s-extra-args", "", "Additional arguments to pass to k3s installer, wrapped in quotes (e.g. --k3s-extra-args '--node-taint key=value:NoExecute')") command.Flags().String("k3s-version", "", "Set a version to install, overrides k3s-channel") @@ -82,6 +84,18 @@ func MakeJoin() *cobra.Command { return err } + var nodeToken string + + nodeTokenPath, _ := command.Flags().GetString("node-token-path") + if len(nodeTokenPath) > 0 { + data, err := os.ReadFile(nodeTokenPath) + if err != nil { + return err + } + + nodeToken = strings.TrimSpace(string(data)) + } + host, err := command.Flags().GetString("host") if err != nil { return err @@ -94,6 +108,7 @@ func MakeJoin() *cobra.Command { if err != nil { return err } + if len(dataDir) == 0 { return fmt.Errorf("--server-data-dir must be set") } @@ -174,94 +189,53 @@ func MakeJoin() *cobra.Command { if useSudo { sudoPrefix = "sudo " } - sshKeyPath := expandPath(sshKey) - address := fmt.Sprintf("%s:%d", serverHost, serverPort) - - var sshOperator *operator.SSHOperator - var initialSSHErr error - if runtime.GOOS != "windows" { - - var sshAgentAuthMethod ssh.AuthMethod - sshAgentAuthMethod, initialSSHErr = sshAgentOnly() - if initialSSHErr == nil { - // Try SSH agent without parsing key files, will succeed if the user - // has already added a key to the SSH Agent, or if using a configured - // smartcard - config := &ssh.ClientConfig{ - User: serverUser, - Auth: []ssh.AuthMethod{sshAgentAuthMethod}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - sshOperator, initialSSHErr = operator.NewSSHOperator(address, config) - } - } else { - initialSSHErr = fmt.Errorf("ssh-agent unsupported on windows") - } - // If the initial connection attempt fails fall through to the using - // the supplied/default private key file - var publicKeyFileAuth ssh.AuthMethod - var closeSSHAgent func() error - if initialSSHErr != nil { - var err error - publicKeyFileAuth, closeSSHAgent, err = loadPublickey(sshKeyPath) - if err != nil { - return fmt.Errorf("unable to load the ssh key with path %q: %w", sshKeyPath, err) + if len(nodeToken) == 0 { + address := fmt.Sprintf("%s:%d", serverHost, serverPort) + + sshOperator, sshOperatorDone, errored, err := connectOperator(serverUser, address, sshKeyPath) + if errored { + return err } - defer closeSSHAgent() + if sshOperatorDone != nil { + defer sshOperatorDone() + } - config := &ssh.ClientConfig{ - User: serverUser, - Auth: []ssh.AuthMethod{ - publicKeyFileAuth, - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + getTokenCommand := fmt.Sprintf("%scat %s\n", sudoPrefix, path.Join(dataDir, "/server/node-token")) + if printCommand { + fmt.Printf("ssh: %s\n", getTokenCommand) } - sshOperator, err = operator.NewSSHOperator(address, config) + streamToStdio := false + res, err := sshOperator.ExecuteStdio(getTokenCommand, streamToStdio) if err != nil { - return fmt.Errorf("unable to connect to (server) %s over ssh: %w", address, err) + return fmt.Errorf("unable to get join-token from server: %w", err) } - } - - defer sshOperator.Close() - getTokenCommand := fmt.Sprintf("%scat %s\n", sudoPrefix, path.Join(dataDir, "/server/node-token")) - if printCommand { - fmt.Printf("ssh: %s\n", getTokenCommand) - } - - streamToStdio := false - res, err := sshOperator.ExecuteStdio(getTokenCommand, streamToStdio) - - if err != nil { - return fmt.Errorf("unable to get join-token from server: %w", err) - } + if len(res.StdErr) > 0 { + fmt.Printf("Error or warning getting node-token: %s\n", res.StdErr) + } else { + fmt.Printf("Received node-token from %s.. ok.\n", serverHost) + } - if len(res.StdErr) > 0 { - fmt.Printf("Error or warning getting node-token: %s\n", res.StdErr) - } else { - fmt.Printf("Received node-token from %s.. ok.\n", serverHost) - } + // Explicit close of the SSH connection as early as possible + // which complements the defer + if sshOperatorDone != nil { + sshOperatorDone() + } - if closeSSHAgent != nil { - closeSSHAgent() + nodeToken = strings.TrimSpace(string(res.StdOut)) } - sshOperator.Close() - - joinToken := strings.TrimSpace(string(res.StdOut)) - if server { tlsSan, _ := command.Flags().GetString("tls-san") - err = setupAdditionalServer(serverHost, host, port, user, sshKeyPath, joinToken, k3sExtraArgs, k3sVersion, k3sChannel, tlsSan, printCommand, serverURL) + err = setupAdditionalServer(serverHost, host, port, user, sshKeyPath, nodeToken, k3sExtraArgs, k3sVersion, k3sChannel, tlsSan, printCommand, serverURL) } else { - err = setupAgent(serverHost, host, port, user, sshKeyPath, joinToken, k3sExtraArgs, k3sVersion, k3sChannel, printCommand, serverURL) + err = setupAgent(serverHost, host, port, user, sshKeyPath, nodeToken, k3sExtraArgs, k3sVersion, k3sChannel, printCommand, serverURL) } if err == nil { diff --git a/cmd/node-token.go b/cmd/node-token.go new file mode 100644 index 00000000..70639581 --- /dev/null +++ b/cmd/node-token.go @@ -0,0 +1,155 @@ +package cmd + +import ( + "fmt" + "net" + "os" + "path" + "strings" + + "github.com/alexellis/k3sup/pkg" + ssh "github.com/alexellis/k3sup/pkg/operator" + + "github.com/spf13/cobra" +) + +// MakeNodeToken creates the node-token command +func MakeNodeToken() *cobra.Command { + var command = &cobra.Command{ + Use: "node-token", + Short: "Retrieve the node token from a server", + Long: `Retrieve the node token from a server required for a +server or agent to join the cluster. + +` + pkg.SupportMessageShort + ` +`, + Example: ` # Get the node token from the server and pipe it to a file + k3sup node-token --ip IP --user USER > token.txt +`, + SilenceUsage: true, + } + + command.Flags().IP("ip", net.ParseIP("127.0.0.1"), "Public IP of node") + command.Flags().String("user", "root", "Username for SSH login") + + command.Flags().String("host", "", "Public hostname of node on which to install agent") + + command.Flags().Bool("local", false, "Use local machine instead of ssh client") + command.Flags().String("ssh-key", "~/.ssh/id_rsa", "The ssh key to use for remote login") + command.Flags().Int("ssh-port", 22, "The port on which to connect for ssh") + command.Flags().Bool("sudo", true, "Use sudo for installation. e.g. set to false when using the root user and no sudo is available.") + + command.Flags().Bool("print-command", false, "Print the command to be executed") + command.Flags().String("server-data-dir", "/var/lib/rancher/k3s/", "Override the path used to fetch the node-token from the server") + + command.PreRunE = func(command *cobra.Command, args []string) error { + local, err := command.Flags().GetBool("local") + if err != nil { + return err + } + + if !local { + _, err = command.Flags().GetString("host") + if err != nil { + return err + } + + if _, err := command.Flags().GetIP("ip"); err != nil { + return err + } + + if _, err := command.Flags().GetInt("ssh-port"); err != nil { + return err + } + } + return nil + } + + command.RunE = func(command *cobra.Command, args []string) error { + + fmt.Fprintf(os.Stderr, "Fetching: /etc/rancher/k3s/k3s.yaml\n") + + useSudo, err := command.Flags().GetBool("sudo") + if err != nil { + return err + } + + sudoPrefix := "" + if useSudo { + sudoPrefix = "sudo " + } + + local, _ := command.Flags().GetBool("local") + + ip, err := command.Flags().GetIP("ip") + if err != nil { + return err + } + + host, err := command.Flags().GetString("host") + if err != nil { + return err + } + if len(host) == 0 { + host = ip.String() + } + + port, _ := command.Flags().GetInt("ssh-port") + user, _ := command.Flags().GetString("user") + sshKey, _ := command.Flags().GetString("ssh-key") + + dataDir, _ := command.Flags().GetString("server-data-dir") + + sshKeyPath := expandPath(sshKey) + address := fmt.Sprintf("%s:%d", host, port) + if !local { + fmt.Fprintf(os.Stderr, "Remote: %s\n", address) + } + + printCommand := false + + getTokenCommand := fmt.Sprintf("%scat %s\n", sudoPrefix, path.Join(dataDir, "/server/node-token")) + if printCommand { + fmt.Printf("ssh: %s\n", getTokenCommand) + } + + var operator ssh.CommandOperator + if local { + operator = ssh.ExecOperator{} + } else { + sshOperator, sshOperatorDone, errored, err := connectOperator(user, address, sshKeyPath) + if errored { + return err + } + operator = sshOperator + + if sshOperatorDone != nil { + defer sshOperatorDone() + } + } + + nodeToken, err := obtainNodeToken(operator, getTokenCommand, host) + if err != nil { + return err + } + + if len(nodeToken) == 0 { + return fmt.Errorf("no node token found") + } + + fmt.Println(nodeToken) + return nil + } + + return command +} + +func obtainNodeToken(operator ssh.CommandOperator, command, host string) (string, error) { + res, err := operator.ExecuteStdio(command, false) + if err != nil { + return "", fmt.Errorf("error received processing command: %s", err) + } + + return strings.TrimSpace(string(res.StdOut)), nil + +} diff --git a/cmd/plan.go b/cmd/plan.go new file mode 100644 index 00000000..5eaf7872 --- /dev/null +++ b/cmd/plan.go @@ -0,0 +1,152 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/alexellis/k3sup/pkg" + "github.com/spf13/cobra" +) + +func MakePlan() *cobra.Command { + var command = &cobra.Command{ + Use: "plan", + Short: "Plan an installation of K3s.", + Long: `Generate a plan of installation commands for K3s for a HA cluster. + +Format: + +[{ + "hostname": "node-1", + "ip": "192.168.1.128" +}] + +` + pkg.SupportMessageShort + ` +`, + Example: ` Generate an installation script where 3x of the + available hosts are dedicated as servers. + k3sup plan hosts.json --servers 3 +`, + SilenceUsage: true, + } + + command.Flags().Int("servers", 3, "Number of servers to use from the pool of hosts") + command.Flags().String("local-path", "kubeconfig", "Where to save the kubeconfig file") + command.Flags().String("context", "default", "Name of the kubeconfig context to use") + command.Flags().String("user", "root", "Username for SSH login") + + command.Flags().String("ssh-key", "", "Path to the private key for SSH login") + command.Flags().String("tls-san", "", "SAN for TLS certificates, can be a comma-separated list") + + // Background + command.Flags().Bool("background", false, "Run the installation in the background for all agents/nodes after the first server is up") + + command.Flags().Int("limit", 0, "Maximum number of nodes to use from the pool of hosts, 0 for all") + + command.RunE = func(cmd *cobra.Command, args []string) error { + + if len(args) == 0 { + return fmt.Errorf("give a path to a JSON file containing a list of hosts") + } + + nodeLimit, _ := cmd.Flags().GetInt("limit") + name := args[0] + data, err := os.ReadFile(name) + if err != nil { + return err + } + + background, _ := cmd.Flags().GetBool("background") + + var hosts []Host + if err = json.Unmarshal(data, &hosts); err != nil { + return err + } + + servers, _ := cmd.Flags().GetInt("servers") + kubeconfig, _ := cmd.Flags().GetString("local-path") + contextName, _ := cmd.Flags().GetString("context") + user, _ := cmd.Flags().GetString("user") + tlsSan, _ := cmd.Flags().GetString("tls-san") + + tlsSanStr := "" + if len(tlsSan) > 0 { + tlsSanStr = fmt.Sprintf(` \ +--tls-san %s`, tlsSan) + } + // sshKey, _ := cmd.Flags().GetString("ssh-key") + + bgStr := "" + if background { + bgStr = " &" + } + + serversAdded := 0 + var primaryServer Host + script := "#!/bin/sh\n\n" + + for i, host := range hosts { + if serversAdded == 0 { + + script += `echo ""Setting up primary server 1 +` + + script += fmt.Sprintf(`k3sup install --host %s \ +--user %s \ +--cluster \ +--local-path %s \ +--context %s%s +`, + host.IP, + user, + kubeconfig, + contextName, tlsSanStr) + + script += fmt.Sprintf(` +echo "Saving the server's node-token to ./token.txt" + +k3sup node-token --host %s \ +--user %s > token.txt +`, host.IP, user) + + serversAdded = 1 + primaryServer = host + } else if serversAdded < servers { + script += fmt.Sprintf("\necho \"Setting up additional server: %d\"\n", serversAdded+1) + + script += fmt.Sprintf(`k3sup join --host %s \ +--server-host %s \ +--server \ +--node-token-path ./token.txt \ +--user %s%s%s +`, host.IP, primaryServer.IP, user, tlsSanStr, bgStr) + + serversAdded++ + } else { + script += fmt.Sprintf("\necho \"Setting up worker: %d\"\n", (i+1)-serversAdded) + + script += fmt.Sprintf(`k3sup join --host %s \ +--server-host %s \ +--node-token-path ./token.txt \ +--user %s%s +`, host.IP, primaryServer.IP, user, bgStr) + } + + if nodeLimit > 0 && i+1 >= nodeLimit { + break + } + } + + fmt.Printf("%s\n", script) + + return nil + } + + return command +} + +type Host struct { + Hostname string `json:"hostname"` + IP string `json:"ip"` +} diff --git a/go.mod b/go.mod index 5fe1a153..7996686c 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,11 @@ require ( github.com/morikuni/aec v1.0.0 github.com/spf13/cobra v1.7.0 golang.org/x/crypto v0.13.0 + golang.org/x/term v0.12.0 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect ) diff --git a/main.go b/main.go index 194a8962..02150175 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,8 @@ func main() { cmdJoin := cmd.MakeJoin() cmdUpdate := cmd.MakeUpdate() cmdReady := cmd.MakeReady() + cmdPlan := cmd.MakePlan() + cmdNodeToken := cmd.MakeNodeToken() printk3supASCIIArt := cmd.PrintK3supASCIIArt @@ -54,6 +56,8 @@ func main() { rootCmd.AddCommand(cmdJoin) rootCmd.AddCommand(cmdUpdate) rootCmd.AddCommand(cmdReady) + rootCmd.AddCommand(cmdPlan) + rootCmd.AddCommand(cmdNodeToken) if err := rootCmd.Execute(); err != nil { os.Exit(1)