From bf52588956fb9c27c146d82489b91bb429f4cb0e Mon Sep 17 00:00:00 2001 From: Stephen Provine Date: Thu, 16 Dec 2021 17:13:29 -0800 Subject: [PATCH] Add buildkit + containerd support (#69) --- README.md | 11 +++-- cli/kdo/kdo.go | 38 +++++++++++---- pkg/buildctl/cli.go | 56 ++++++++++++++++++++++ pkg/imagebuild/imagebuild.go | 92 ++++++++++++++++++++++++++++-------- pkg/pod/apply.go | 32 ++++++++++--- pkg/server/server.go | 85 ++++++++++++++++++++++++++------- 6 files changed, 261 insertions(+), 53 deletions(-) create mode 100644 pkg/buildctl/cli.go diff --git a/README.md b/README.md index 7253947..abeba5a 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Kdo can also be used for longer-running connected development sessions where loc ## Prerequisites and Installation -Kdo requires the `kubectl` CLI to communicate with a Kubernetes cluster and the `docker` CLI to perform dynamic image builds, so first make sure you have these installed and available in your PATH. Then, download the latest [release](https://github.com/stepro/kdo/releases) for your platform and add the `kdo` binary to your PATH. +Kdo requires the `kubectl` CLI to communicate with a Kubernetes cluster and the `docker` or `buildctl` CLIs to perform dynamic image builds, so first make sure you have these installed and available in your PATH. Then, download the latest [release](https://github.com/stepro/kdo/releases) for your platform and add the `kdo` binary to your PATH. By default `kdo` utilizes the current `kubectl` context, so point it at the Kubernetes cluster of your choice and you're good to go! @@ -120,10 +120,13 @@ The scope flag (`--scope`) can be used to change how Kubernetes cluster resource ### Build flags -These flags customize how the `docker` CLI is used when building images. +These flags customize how the `docker` or `buildctl` CLIs are used when building images. Flag | Default | Description ---- | ------- | ----------- +`--builder` | `docker` | the image builder to use +`--buildctl` | `buildctl` | path to the buildctl CLI +`--buildctl-debug` | `false` | the buildctl CLI debug flag `--docker` | `docker` | path to the docker CLI `--docker-config` | `` | path to the docker CLI config files `--docker-log-level` | `` | the docker CLI logging level @@ -131,7 +134,9 @@ Flag | Default | Description `--build-arg` | `[]` | build-time variables in the form `name=value` `--build-target` | `` | dockerfile target to build -When the `docker` CLI is invoked, it does not use the default configured Docker daemon. Instead, it uses the kdo server components to directly access the Docker daemon running on a node in the Kubernetes cluster. Therefore, it is theoretically not a requirement that the local machine is actually running Docker, although in most cases (e.g. Docker Desktop) this will be the case. It **is**, however, a requirement that the node on which the kdo pod is scheduled is using Docker for its container runtime and the Docker daemon socket at `/var/run/docker.sock` on the host can be volume mounted into the pod. +The `buildkit` builder should be chosen when the Kubernetes cluster nodes use containerd to run containers. It requires the `buildctl` CLI to be installed locally which is configured to communicate with a buildkitd daemon run by the kdo server components, which in turn is configured to communicate with the containerd daemon. + +The `docker` builder should be chosen when the Kubernetes cluster nodes use Docker to run containers. It requires the `docker` CLI to be installed locally which is configured to communicate with the Docker daemon running on a node in the Kubernetes cluster. ### Configuration flags diff --git a/cli/kdo/kdo.go b/cli/kdo/kdo.go index c4e2a3b..d228868 100644 --- a/cli/kdo/kdo.go +++ b/cli/kdo/kdo.go @@ -14,6 +14,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/stepro/kdo/pkg/buildctl" "github.com/stepro/kdo/pkg/docker" "github.com/stepro/kdo/pkg/filesync" "github.com/stepro/kdo/pkg/imagebuild" @@ -28,7 +29,7 @@ import ( var cmd = &cobra.Command{ Short: "Kdo: deployless development on Kubernetes", Use: usage, - Version: "0.7.0", + Version: "0.8.0", Example: examples, RunE: run, } @@ -84,6 +85,11 @@ var flags struct { uninstall bool scope string build struct { + builder string + buildctl struct { + path string + buildctl.Options + } docker struct { path string docker.Options @@ -155,6 +161,12 @@ func init() { "scope", "", "scoping identifier for cluster resources") // Build flags + cmd.Flags().StringVar(&flags.build.builder, + "builder", "docker", "the image builder to use") + cmd.Flags().StringVar(&flags.build.buildctl.path, + "buildctl", "buildctl", "path to the buildctl CLI") + cmd.Flags().BoolVar(&flags.build.buildctl.Debug, + "buildctl-debug", false, "the buildctl CLI debug flag") cmd.Flags().StringVar(&flags.build.docker.path, "docker", "docker", "path to the docker CLI") cmd.Flags().StringVar(&flags.build.docker.Config, @@ -448,7 +460,7 @@ func run(cmd *cobra.Command, args []string) error { hash = fmt.Sprintf("%s\n%s\n%s", flags.scope, hash, flags.config.inherit) hash = fmt.Sprintf("%x", sha1.Sum([]byte(hash)))[:16] if buildDir != "" { - image = fmt.Sprintf("kdo-%s:%d", hash, time.Now().UnixNano()) + image = fmt.Sprintf("dev.local/kdo-%s:%d", hash, time.Now().UnixNano()) } command := args[1:] @@ -485,12 +497,22 @@ func run(cmd *cobra.Command, args []string) error { var build func(pod string) error if buildDir != "" { - build = func(pod string) error { - d := docker.NewCLI( - flags.build.docker.path, - &flags.build.docker.Options, - out, output.LevelVerbose) - return imagebuild.Build(k, pod, d, &flags.build.Options, image, buildDir, out) + if flags.build.builder == "buildkit" { + build = func(pod string) error { + bc := buildctl.NewCLI( + flags.build.buildctl.path, + &flags.build.buildctl.Options, + out, output.LevelVerbose) + return imagebuild.Build(k, pod, bc, nil, &flags.build.Options, image, buildDir, out) + } + } else if flags.build.builder == "docker" { + build = func(pod string) error { + d := docker.NewCLI( + flags.build.docker.path, + &flags.build.docker.Options, + out, output.LevelVerbose) + return imagebuild.Build(k, pod, nil, d, &flags.build.Options, image, buildDir, out) + } } } diff --git a/pkg/buildctl/cli.go b/pkg/buildctl/cli.go new file mode 100644 index 0000000..3a1cbf0 --- /dev/null +++ b/pkg/buildctl/cli.go @@ -0,0 +1,56 @@ +package buildctl + +import ( + "os/exec" + + "github.com/stepro/kdo/pkg/command" + "github.com/stepro/kdo/pkg/output" +) + +// Options represents global options for the buildctl CLI +type Options struct { + Debug bool +} + +// CLI represents the buildctl CLI +type CLI interface { + // EachLine runs a buildctl command that sends + // its lines of standard error to a callback + EachErrLine(args []string, fn func(line string)) error +} + +type cli struct { + path string + opt *Options + out *output.Interface + verb output.Level +} + +func (d *cli) command(arg ...string) *exec.Cmd { + cmd := exec.Command(d.path) + + var globalOptions []string + if d.opt.Debug { + globalOptions = append(globalOptions, "--debug") + } + cmd.Args = append(cmd.Args, append(globalOptions, arg...)...) + + return cmd +} + +func (d *cli) EachErrLine(args []string, fn func(line string)) error { + cmd := d.command(args...) + cmd.Stderr = output.NewLineWriter(fn) + + return command.Run(cmd, d.out, d.verb) +} + +// NewCLI creates a new buildctl CLI object +func NewCLI(path string, options *Options, out *output.Interface, verb output.Level) CLI { + return &cli{ + path: path, + opt: options, + out: out, + verb: verb, + } +} diff --git a/pkg/imagebuild/imagebuild.go b/pkg/imagebuild/imagebuild.go index b03d297..a87b466 100644 --- a/pkg/imagebuild/imagebuild.go +++ b/pkg/imagebuild/imagebuild.go @@ -2,9 +2,12 @@ package imagebuild import ( "fmt" + "path/filepath" + "regexp" "strings" "time" + "github.com/stepro/kdo/pkg/buildctl" "github.com/stepro/kdo/pkg/docker" "github.com/stepro/kdo/pkg/kubectl" "github.com/stepro/kdo/pkg/output" @@ -27,7 +30,7 @@ type Options struct { } // Build builds an image on the node that is running a pod -func Build(k kubectl.CLI, pod string, d docker.CLI, options *Options, image string, context string, out *output.Interface) error { +func Build(k kubectl.CLI, pod string, bc buildctl.CLI, d docker.CLI, options *Options, image string, context string, out *output.Interface) error { return pkgerror(out.Do("Building image", func(op output.Operation) error { op.Progress("determining build node") var node string @@ -51,32 +54,81 @@ func Build(k kubectl.CLI, pod string, d docker.CLI, options *Options, image stri return fmt.Errorf("cannot build on node %s", node) } - op.Progress("connecting to docker daemon") - dockerPort, stop, err := portforward.StartOne(k, "kube-system", nodePods[node], "2375") + if bc != nil { + op.Progress("connecting to buildkit daemon") + } else { + op.Progress("connecting to docker daemon") + } + builderPort, stop, err := portforward.StartOne(k, "kube-system", nodePods[node], "2375") if err != nil { return err } defer stop() - buildArgs := []string{"--host", "localhost:" + dockerPort, "build"} - if options.File != "" { - buildArgs = append(buildArgs, "--file", options.File) - } - for _, arg := range options.Args { - buildArgs = append(buildArgs, "--build-arg", arg) - } - if options.Target != "" { - buildArgs = append(buildArgs, "--target", options.Target) + var buildArgs []string + if bc != nil { + buildArgs = []string{ + "--addr", "tcp://localhost:" + builderPort, + "build", + "--frontend", "dockerfile.v0", + "--local", "context=" + context, + "--local", "dockerfile=" + context, + } + if options.File == "" { + buildArgs = append(buildArgs, "--local", "dockerfile="+context) + } else { + dir, file := filepath.Split(options.File) + if dir == "" { + buildArgs = append(buildArgs, "--local", "dockerfile="+context) + } else { + buildArgs = append(buildArgs, "--local", "dockerfile="+dir) + } + buildArgs = append(buildArgs, "--opt", "filename="+file) + } + for _, arg := range options.Args { + buildArgs = append(buildArgs, "--opt", "build-arg:"+arg) + } + if options.Target != "" { + buildArgs = append(buildArgs, "--opt", "target="+options.Target) + } + buildArgs = append(buildArgs, "--output", "type=image,name="+image+",unpack=true") + } else /* if d != nil */ { + buildArgs = []string{ + "--host", "localhost:" + builderPort, + "build", + } + if options.File != "" { + buildArgs = append(buildArgs, "--file", options.File) + } + for _, arg := range options.Args { + buildArgs = append(buildArgs, "--build-arg", arg) + } + if options.Target != "" { + buildArgs = append(buildArgs, "--target", options.Target) + } + buildArgs = append(buildArgs, "--tag", image, context) } - buildArgs = append(buildArgs, "--tag", image, context) op.Progress("running") - return d.EachLine(buildArgs, func(line string) { - if out.Level < output.LevelVerbose && (strings.HasPrefix(line, "Sending build context ") || strings.HasPrefix(line, "Step ")) { - op.Progress("s" + line[1:]) - } else { - out.Verbose("[docker] %s", line) - } - }) + if bc != nil { + re := regexp.MustCompile(`^#[0-9]+\s(\[.*)$`) + return bc.EachErrLine(buildArgs, func(line string) { + if out.Level < output.LevelVerbose { + if matches := re.FindStringSubmatch(line); len(matches) > 0 { + op.Progress(matches[1]) + } + } else { + out.Verbose("[buildctl] %s", line) + } + }) + } else { + return d.EachLine(buildArgs, func(line string) { + if out.Level < output.LevelVerbose && (strings.HasPrefix(line, "Sending build context ") || strings.HasPrefix(line, "Step ")) { + op.Progress("s" + line[1:]) + } else { + out.Verbose("[docker] %s", line) + } + }) + } })) } diff --git a/pkg/pod/apply.go b/pkg/pod/apply.go index d97466f..4179ac4 100644 --- a/pkg/pod/apply.go +++ b/pkg/pod/apply.go @@ -126,23 +126,43 @@ func Apply(k kubectl.CLI, hash string, config *Config, build func(pod string) er } if build != nil { spec.appendobj("volumes", map[string]interface{}{ - "name": "kdo-docker-socket", + "name": "kdo-host-run-containerd", "hostPath": map[string]interface{}{ - "path": "/var/run/docker.sock", + "path": "/run/containerd", + }, + }).appendobj("volumes", map[string]interface{}{ + "name": "kdo-host-run-docker-sock", + "hostPath": map[string]interface{}{ + // "type": "SocketOrCreate", + "path": "/run/docker.sock", }, }).appendobj("initContainers", map[string]interface{}{ "name": "kdo-await-image-build", - "image": "docker:19.03", + "image": "docker", "volumeMounts": []map[string]interface{}{ { - "name": "kdo-docker-socket", - "mountPath": "/var/run/docker.sock", + "name": "kdo-host-run-containerd", + "mountPath": "/run/containerd", + }, + { + "name": "kdo-host-run-docker-sock", + "mountPath": "/run/docker.sock", }, }, "command": []string{ "/bin/sh", "-c", - `while [ -z "$(docker images ` + config.Image + ` --format '{{.Repository}}')" ]; do sleep 1; done`, + `if [ -S /run/containerd/containerd.sock ]; then + apk add curl + curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.22.0/crictl-v1.22.0-linux-amd64.tar.gz | tar -xzvf - + while [ -z "$(/crictl --runtime-endpoint unix:///run/containerd/containerd.sock images | grep '` + strings.Replace(config.Image, ":", "\\s*", 1) + `')" ]; do + sleep 1 + done + elif [ -S /var/run/docker.sock ]; then + while [ -z "$(docker images ` + config.Image + ` --format '{{.Repository}}')" ]; do + sleep 1 + done + fi`, }, }) } diff --git a/pkg/server/server.go b/pkg/server/server.go index bbcd7ce..b9d337c 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -25,10 +25,19 @@ metadata: labels: component: kdo-server data: - docker-daemon.sh: |- + buildkitd.toml: |- + [worker.containerd] + namespace = "k8s.io" + entrypoint.sh: |- #!/bin/sh - apk add --no-cache socat - exec socat -d tcp4-listen:2375,fork UNIX-CONNECT:/var/run/docker.sock + if [ -e "/run/containerd/containerd.sock" ]; then + exec buildkitd --addr tcp://0.0.0.0:2375 \ + --root /var/lib/kdo/buildkit \ + --oci-worker false --containerd-worker true + elif [ -e "/run/docker.sock" ]; then + apk add --no-cache socat + exec socat -d tcp4-listen:2375,fork UNIX-CONNECT:/run/docker.sock + fi --- apiVersion: apps/v1 kind: DaemonSet @@ -46,29 +55,73 @@ spec: component: kdo-server spec: nodeSelector: - beta.kubernetes.io/os: linux + kubernetes.io/os: linux volumes: + - name: host-run-containerd + hostPath: + type: DirectoryOrCreate + path: /run/containerd + - name: host-run-docker-sock + hostPath: + # type: SocketOrCreate + path: /run/docker.sock + - name: host-tmp + hostPath: + type: Directory + path: /tmp + - name: host-var-lib-buildkit + hostPath: + type: DirectoryOrCreate + path: /var/lib/kdo/buildkit + - name: host-var-lib-containerd + hostPath: + type: DirectoryOrCreate + path: /var/lib/containerd + - name: host-var-log + hostPath: + type: Directory + path: /var/log - name: config configMap: name: kdo-server items: - - key: docker-daemon.sh - path: docker-daemon.sh + - key: buildkitd.toml + path: buildkitd.toml + mode: 0644 + - key: entrypoint.sh + path: entrypoint.sh mode: 0777 - - name: docker-socket - hostPath: - path: /var/run/docker.sock containers: - - name: docker-daemon - image: alpine:3 + - name: kdo-server + image: moby/buildkit volumeMounts: + - name: host-run-containerd + mountPath: /run/containerd + mountPropagation: Bidirectional + - name: host-run-docker-sock + mountPath: /run/docker.sock + - name: host-tmp + mountPath: /tmp + mountPropagation: Bidirectional + - name: host-var-lib-buildkit + mountPath: /var/lib/kdo/buildkit + mountPropagation: Bidirectional + - name: host-var-lib-containerd + mountPath: /var/lib/containerd + mountPropagation: Bidirectional + - name: host-var-log + mountPath: /var/log + mountPropagation: Bidirectional + - name: config + subPath: buildkitd.toml + mountPath: /etc/buildkit/buildkitd.toml - name: config - subPath: docker-daemon.sh - mountPath: /docker-daemon.sh - - name: docker-socket - mountPath: /var/run/docker.sock + subPath: entrypoint.sh + mountPath: /entrypoint.sh + securityContext: + privileged: true command: - - /docker-daemon.sh + - /entrypoint.sh readinessProbe: tcpSocket: port: 2375