Add buildkit + containerd support (#69)

README.md

@@ -16,7 +16,7 @@ Kdo can also be used for longer-running connected development sessions where loc
 
 ## Prerequisites and Installation
 
-Kdo requires the `kubectl` CLI to communicate with a Kubernetes cluster and the `docker` CLI to perform dynamic image builds, so first make sure you have these installed and available in your PATH. Then, download the latest [release](https://github.com/stepro/kdo/releases) for your platform and add the `kdo` binary to your PATH.
+Kdo requires the `kubectl` CLI to communicate with a Kubernetes cluster and the `docker` or `buildctl` CLIs to perform dynamic image builds, so first make sure you have these installed and available in your PATH. Then, download the latest [release](https://github.com/stepro/kdo/releases) for your platform and add the `kdo` binary to your PATH.
 
 By default `kdo` utilizes the current `kubectl` context, so point it at the Kubernetes cluster of your choice and you're good to go!
 
@@ -120,18 +120,23 @@ The scope flag (`--scope`) can be used to change how Kubernetes cluster resource
 
 ### Build flags
 
-These flags customize how the `docker` CLI is used when building images.
+These flags customize how the `docker` or `buildctl` CLIs are used when building images.
 
 Flag | Default | Description
 ---- | ------- | -----------
+`--builder` | `docker` | the image builder to use
+`--buildctl` | `buildctl` | path to the buildctl CLI
+`--buildctl-debug` | `false` | the buildctl CLI debug flag
 `--docker` | `docker` | path to the docker CLI
 `--docker-config` | `<empty>` | path to the docker CLI config files
 `--docker-log-level` | `<empty>` | the docker CLI logging level
 `-f, --build-file` | `<build-dir>/Dockerfile` | dockerfile to build
 `--build-arg` | `[]` | build-time variables in the form `name=value`
 `--build-target` | `<empty>` | dockerfile target to build
 
-When the `docker` CLI is invoked, it does not use the default configured Docker daemon. Instead, it uses the kdo server components to directly access the Docker daemon running on a node in the Kubernetes cluster. Therefore, it is theoretically not a requirement that the local machine is actually running Docker, although in most cases (e.g. Docker Desktop) this will be the case. It **is**, however, a requirement that the node on which the kdo pod is scheduled is using Docker for its container runtime and the Docker daemon socket at `/var/run/docker.sock` on the host can be volume mounted into the pod.
+The `buildkit` builder should be chosen when the Kubernetes cluster nodes use containerd to run containers. It requires the `buildctl` CLI to be installed locally which is configured to communicate with a buildkitd daemon run by the kdo server components, which in turn is configured to communicate with the containerd daemon.
+
+The `docker` builder should be chosen when the Kubernetes cluster nodes use Docker to run containers. It requires the `docker` CLI to be installed locally which is configured to communicate with the Docker daemon running on a node in the Kubernetes cluster.
 
 ### Configuration flags
 

cli/kdo/kdo.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/stepro/kdo/pkg/buildctl"
 	"github.com/stepro/kdo/pkg/docker"
 	"github.com/stepro/kdo/pkg/filesync"
 	"github.com/stepro/kdo/pkg/imagebuild"
@@ -28,7 +29,7 @@ import (
 var cmd = &cobra.Command{
 	Short:   "Kdo: deployless development on Kubernetes",
 	Use:     usage,
-	Version: "0.7.0",
+	Version: "0.8.0",
 	Example: examples,
 	RunE:    run,
 }
@@ -84,6 +85,11 @@ var flags struct {
 	uninstall bool
 	scope     string
 	build     struct {
+		builder  string
+		buildctl struct {
+			path string
+			buildctl.Options
+		}
 		docker struct {
 			path string
 			docker.Options
@@ -155,6 +161,12 @@ func init() {
 		"scope", "", "scoping identifier for cluster resources")
 
 	// Build flags
+	cmd.Flags().StringVar(&flags.build.builder,
+		"builder", "docker", "the image builder to use")
+	cmd.Flags().StringVar(&flags.build.buildctl.path,
+		"buildctl", "buildctl", "path to the buildctl CLI")
+	cmd.Flags().BoolVar(&flags.build.buildctl.Debug,
+		"buildctl-debug", false, "the buildctl CLI debug flag")
 	cmd.Flags().StringVar(&flags.build.docker.path,
 		"docker", "docker", "path to the docker CLI")
 	cmd.Flags().StringVar(&flags.build.docker.Config,
@@ -448,7 +460,7 @@ func run(cmd *cobra.Command, args []string) error {
 	hash = fmt.Sprintf("%s\n%s\n%s", flags.scope, hash, flags.config.inherit)
 	hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash)))[:16]
 	if buildDir != "" {
-		image = fmt.Sprintf("kdo-%s:%d", hash, time.Now().UnixNano())
+		image = fmt.Sprintf("dev.local/kdo-%s:%d", hash, time.Now().UnixNano())
 	}
 	command := args[1:]
 
@@ -485,12 +497,22 @@ func run(cmd *cobra.Command, args []string) error {
 
 	var build func(pod string) error
 	if buildDir != "" {
-		build = func(pod string) error {
-			d := docker.NewCLI(
-				flags.build.docker.path,
-				&flags.build.docker.Options,
-				out, output.LevelVerbose)
-			return imagebuild.Build(k, pod, d, &flags.build.Options, image, buildDir, out)
+		if flags.build.builder == "buildkit" {
+			build = func(pod string) error {
+				bc := buildctl.NewCLI(
+					flags.build.buildctl.path,
+					&flags.build.buildctl.Options,
+					out, output.LevelVerbose)
+				return imagebuild.Build(k, pod, bc, nil, &flags.build.Options, image, buildDir, out)
+			}
+		} else if flags.build.builder == "docker" {
+			build = func(pod string) error {
+				d := docker.NewCLI(
+					flags.build.docker.path,
+					&flags.build.docker.Options,
+					out, output.LevelVerbose)
+				return imagebuild.Build(k, pod, nil, d, &flags.build.Options, image, buildDir, out)
+			}
 		}
 	}
 

pkg/buildctl/cli.go

@@ -0,0 +1,56 @@
+package buildctl
+
+import (
+	"os/exec"
+
+	"github.com/stepro/kdo/pkg/command"
+	"github.com/stepro/kdo/pkg/output"
+)
+
+// Options represents global options for the buildctl CLI
+type Options struct {
+	Debug bool
+}
+
+// CLI represents the buildctl CLI
+type CLI interface {
+	// EachLine runs a buildctl command that sends
+	// its lines of standard error to a callback
+	EachErrLine(args []string, fn func(line string)) error
+}
+
+type cli struct {
+	path string
+	opt  *Options
+	out  *output.Interface
+	verb output.Level
+}
+
+func (d *cli) command(arg ...string) *exec.Cmd {
+	cmd := exec.Command(d.path)
+
+	var globalOptions []string
+	if d.opt.Debug {
+		globalOptions = append(globalOptions, "--debug")
+	}
+	cmd.Args = append(cmd.Args, append(globalOptions, arg...)...)
+
+	return cmd
+}
+
+func (d *cli) EachErrLine(args []string, fn func(line string)) error {
+	cmd := d.command(args...)
+	cmd.Stderr = output.NewLineWriter(fn)
+
+	return command.Run(cmd, d.out, d.verb)
+}
+
+// NewCLI creates a new buildctl CLI object
+func NewCLI(path string, options *Options, out *output.Interface, verb output.Level) CLI {
+	return &cli{
+		path: path,
+		opt:  options,
+		out:  out,
+		verb: verb,
+	}
+}

pkg/imagebuild/imagebuild.go

@@ -2,9 +2,12 @@ package imagebuild
 
 import (
 	"fmt"
+	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
+	"github.com/stepro/kdo/pkg/buildctl"
 	"github.com/stepro/kdo/pkg/docker"
 	"github.com/stepro/kdo/pkg/kubectl"
 	"github.com/stepro/kdo/pkg/output"
@@ -27,7 +30,7 @@ type Options struct {
 }
 
 // Build builds an image on the node that is running a pod
-func Build(k kubectl.CLI, pod string, d docker.CLI, options *Options, image string, context string, out *output.Interface) error {
+func Build(k kubectl.CLI, pod string, bc buildctl.CLI, d docker.CLI, options *Options, image string, context string, out *output.Interface) error {
 	return pkgerror(out.Do("Building image", func(op output.Operation) error {
 		op.Progress("determining build node")
 		var node string
@@ -51,32 +54,81 @@ func Build(k kubectl.CLI, pod string, d docker.CLI, options *Options, image stri
 			return fmt.Errorf("cannot build on node %s", node)
 		}
 
-		op.Progress("connecting to docker daemon")
-		dockerPort, stop, err := portforward.StartOne(k, "kube-system", nodePods[node], "2375")
+		if bc != nil {
+			op.Progress("connecting to buildkit daemon")
+		} else {
+			op.Progress("connecting to docker daemon")
+		}
+		builderPort, stop, err := portforward.StartOne(k, "kube-system", nodePods[node], "2375")
 		if err != nil {
 			return err
 		}
 		defer stop()
 
-		buildArgs := []string{"--host", "localhost:" + dockerPort, "build"}
-		if options.File != "" {
-			buildArgs = append(buildArgs, "--file", options.File)
-		}
-		for _, arg := range options.Args {
-			buildArgs = append(buildArgs, "--build-arg", arg)
-		}
-		if options.Target != "" {
-			buildArgs = append(buildArgs, "--target", options.Target)
+		var buildArgs []string
+		if bc != nil {
+			buildArgs = []string{
+				"--addr", "tcp://localhost:" + builderPort,
+				"build",
+				"--frontend", "dockerfile.v0",
+				"--local", "context=" + context,
+				"--local", "dockerfile=" + context,
+			}
+			if options.File == "" {
+				buildArgs = append(buildArgs, "--local", "dockerfile="+context)
+			} else {
+				dir, file := filepath.Split(options.File)
+				if dir == "" {
+					buildArgs = append(buildArgs, "--local", "dockerfile="+context)
+				} else {
+					buildArgs = append(buildArgs, "--local", "dockerfile="+dir)
+				}
+				buildArgs = append(buildArgs, "--opt", "filename="+file)
+			}
+			for _, arg := range options.Args {
+				buildArgs = append(buildArgs, "--opt", "build-arg:"+arg)
+			}
+			if options.Target != "" {
+				buildArgs = append(buildArgs, "--opt", "target="+options.Target)
+			}
+			buildArgs = append(buildArgs, "--output", "type=image,name="+image+",unpack=true")
+		} else /* if d != nil */ {
+			buildArgs = []string{
+				"--host", "localhost:" + builderPort,
+				"build",
+			}
+			if options.File != "" {
+				buildArgs = append(buildArgs, "--file", options.File)
+			}
+			for _, arg := range options.Args {
+				buildArgs = append(buildArgs, "--build-arg", arg)
+			}
+			if options.Target != "" {
+				buildArgs = append(buildArgs, "--target", options.Target)
+			}
+			buildArgs = append(buildArgs, "--tag", image, context)
 		}
-		buildArgs = append(buildArgs, "--tag", image, context)
 
 		op.Progress("running")
-		return d.EachLine(buildArgs, func(line string) {
-			if out.Level < output.LevelVerbose && (strings.HasPrefix(line, "Sending build context ") || strings.HasPrefix(line, "Step ")) {
-				op.Progress("s" + line[1:])
-			} else {
-				out.Verbose("[docker] %s", line)
-			}
-		})
+		if bc != nil {
+			re := regexp.MustCompile(`^#[0-9]+\s(\[.*)$`)
+			return bc.EachErrLine(buildArgs, func(line string) {
+				if out.Level < output.LevelVerbose {
+					if matches := re.FindStringSubmatch(line); len(matches) > 0 {
+						op.Progress(matches[1])
+					}
+				} else {
+					out.Verbose("[buildctl] %s", line)
+				}
+			})
+		} else {
+			return d.EachLine(buildArgs, func(line string) {
+				if out.Level < output.LevelVerbose && (strings.HasPrefix(line, "Sending build context ") || strings.HasPrefix(line, "Step ")) {
+					op.Progress("s" + line[1:])
+				} else {
+					out.Verbose("[docker] %s", line)
+				}
+			})
+		}
 	}))
 }

pkg/pod/apply.go

@@ -126,23 +126,43 @@ func Apply(k kubectl.CLI, hash string, config *Config, build func(pod string) er
 			}
 			if build != nil {
 				spec.appendobj("volumes", map[string]interface{}{
-					"name": "kdo-docker-socket",
+					"name": "kdo-host-run-containerd",
 					"hostPath": map[string]interface{}{
-						"path": "/var/run/docker.sock",
+						"path": "/run/containerd",
+					},
+				}).appendobj("volumes", map[string]interface{}{
+					"name": "kdo-host-run-docker-sock",
+					"hostPath": map[string]interface{}{
+						// "type": "SocketOrCreate",
+						"path": "/run/docker.sock",
 					},
 				}).appendobj("initContainers", map[string]interface{}{
 					"name":  "kdo-await-image-build",
-					"image": "docker:19.03",
+					"image": "docker",
 					"volumeMounts": []map[string]interface{}{
 						{
-							"name":      "kdo-docker-socket",
-							"mountPath": "/var/run/docker.sock",
+							"name":      "kdo-host-run-containerd",
+							"mountPath": "/run/containerd",
+						},
+						{
+							"name":      "kdo-host-run-docker-sock",
+							"mountPath": "/run/docker.sock",
 						},
 					},
 					"command": []string{
 						"/bin/sh",
 						"-c",
-						`while [ -z "$(docker images ` + config.Image + ` --format '{{.Repository}}')" ]; do sleep 1; done`,
+						`if [ -S /run/containerd/containerd.sock ]; then
+                          apk add curl
+                          curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.22.0/crictl-v1.22.0-linux-amd64.tar.gz | tar -xzvf -
+                          while [ -z "$(/crictl --runtime-endpoint unix:///run/containerd/containerd.sock images | grep '` + strings.Replace(config.Image, ":", "\\s*", 1) + `')" ]; do
+						  	sleep 1
+						  done
+                        elif [ -S /var/run/docker.sock ]; then
+                          while [ -z "$(docker images ` + config.Image + ` --format '{{.Repository}}')" ]; do
+						  	sleep 1
+						  done
+                        fi`,
 					},
 				})
 			}