Merge pull request #188 from phylake/build-args

Build args

CHANGELOG.md

@@ -1,5 +1,10 @@
 `porter` is [semantically versioned](http://semver.org/spec/v2.0.0.html)
 
+### v5.2.0
+
+- added support for build args in the builder container
+- build porter with Go 1.10.3
+
 ### v5.1.0
 
 - build porter with Go 1.10.1

Dockerfile.godep

@@ -1,3 +1,3 @@
-FROM golang:1.10.1
+FROM golang:1.10.3
 
 RUN go get github.com/tools/godep

Dockerfile.linux

@@ -1,4 +1,4 @@
-FROM golang:1.10.1
+FROM golang:1.10.3
 
 ADD . /go/src/github.com/adobe-platform/porter
 WORKDIR /go/src/github.com/adobe-platform/porter

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM golang:1.10.1
+FROM golang:1.10.3
 
 RUN go get github.com/onsi/ginkgo/ginkgo
 RUN go get github.com/onsi/gomega

RELEASE_NOTES.md

@@ -2,6 +2,11 @@ See the [CHANGELOG](CHANGELOG.md) for a complete list of changes.
 
 `porter` is [semantically versioned](http://semver.org/spec/v2.0.0.html)
 
+v5.2
+====
+
+Added support for build args in the [builder container](docs/detailed_design/docker-builder-pattern.md#build-args)
+
 v5.1
 ====
 

conf/conf.go

@@ -73,17 +73,18 @@ type (
 	}
 
 	Container struct {
-		Name            string `yaml:"name"`
-		OriginalName    string
-		Topology        string       `yaml:"topology"`
-		InetPort        int          `yaml:"inet_port"`
-		Uid             *int         `yaml:"uid"`
-		ReadOnly        *bool        `yaml:"read_only"`
-		Dockerfile      string       `yaml:"dockerfile"`
-		DockerfileBuild string       `yaml:"dockerfile_build"`
-		HealthCheck     *HealthCheck `yaml:"health_check"`
-		SrcEnvFile      *SrcEnvFile  `yaml:"src_env_file"`
-		PidsLimit       int          `yaml:"pids_limit"`
+		Name                   string `yaml:"name"`
+		OriginalName           string
+		Topology               string       `yaml:"topology"`
+		InetPort               int          `yaml:"inet_port"`
+		Uid                    *int         `yaml:"uid"`
+		ReadOnly               *bool        `yaml:"read_only"`
+		Dockerfile             string       `yaml:"dockerfile"`
+		DockerfileBuild        string       `yaml:"dockerfile_build"`
+		ValidateBuildArgsExist *bool        `yaml:"validate_build_args_exist"`
+		HealthCheck            *HealthCheck `yaml:"health_check"`
+		SrcEnvFile             *SrcEnvFile  `yaml:"src_env_file"`
+		PidsLimit              int          `yaml:"pids_limit"`
 	}
 
 	SrcEnvFile struct {

docs/detailed_design/docker-builder-pattern.md

@@ -27,6 +27,56 @@ docker run --rm foo-builder | docker build -t foo -f Dockerfile -
 This enables separating a build environment from a runtime environment, as well
 as simply turning the `.dockerignore` blacklist into a `tar` stream whitelist.
 
+Build args
+----------
+
+Often a build environment needs credentials to call into supporting services
+like privately hosted registries or repositories. These credentials can be passed
+to the build container as [build args](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg).
+
+From the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/#arg)
+
+> Warning: It is not recommended to use build-time variables for passing secrets
+> like github keys, user credentials etc. Build-time variable values are visible
+> to any user of the image with the docker history command.
+
+While build args are not supposed to be used for secrets the build container
+isn't the final container. Its contents are available only in `docker history`
+of the machine that runs `porter build pack`, NOT in the final container (unless
+the user passes them on, of course). A `docker rmi -f` is run after the build
+image is built and run, even if intermediate steps fail.
+
+> Also, these values don’t persist in the intermediate or final images like ENV values do.
+
+If all of the above is acceptable then read on.
+
+Porter will take the intersection of (1) build args declared in the
+`Dockerfile.build` (or `dockerfile_build`) and (2) its own environment.
+
+**Example `Dockerfile.build`**
+
+```Dockerfile
+FROM ubuntu:18.04
+
+ARG username
+ARG password
+
+RUN download_dependencies -u $username -p $password
+
+# ...
+```
+
+Porter parses each `ARG` value and performs a [`os.LookupEnv`](https://golang.org/pkg/os/#LookupEnv).
+Environment variables may be empty but they must exist.
+
+If the environment variable `username` is `Bob` and `password` is `easytocrack`
+then the resulting commands would look like this:
+
+```
+docker build -t foo-builder -f Dockerfile.build --build-arg username=Bob --build-arg password=easytocrack .
+docker run --rm foo-builder | docker build -t foo -f Dockerfile -
+```
+
 Use cases
 ---------
 

provision/pack.go

@@ -12,6 +12,7 @@
 package provision
 
 import (
+	"bufio"
 	"crypto/md5"
 	"crypto/rand"
 	"encoding/hex"
@@ -21,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -30,7 +32,10 @@ import (
 	"gopkg.in/inconshreveable/log15.v2"
 )
 
-var dockerSaveLock sync.Mutex
+var (
+	dockerSaveLock sync.Mutex
+	buildArgRe     = regexp.MustCompile(`^ARG (.+)$`)
+)
 
 // Package creates the service payload to deliver to S3
 func Package(log log15.Logger, config *conf.Config) (success bool) {
@@ -117,8 +122,7 @@ func Package(log log15.Logger, config *conf.Config) (success bool) {
 
 		go func(container *conf.Container) {
 
-			successChan <- buildContainer(log, container.Name,
-				container.Dockerfile, container.DockerfileBuild)
+			successChan <- buildContainer(log, container)
 
 		}(container)
 	}
@@ -172,28 +176,71 @@ func Package(log log15.Logger, config *conf.Config) (success bool) {
 	return
 }
 
-func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild string) (success bool) {
+func buildContainer(log log15.Logger, container *conf.Container) (success bool) {
 
-	log = log.New("ImageTag", containerName)
+	log = log.New("ImageTag", container.Name)
 
-	imagePath := fmt.Sprintf("%s/%s.docker", constants.PayloadWorkingDir, containerName)
+	imagePath := fmt.Sprintf("%s/%s.docker", constants.PayloadWorkingDir, container.Name)
 
-	_, err := os.Stat(dockerfile)
+	_, err := os.Stat(container.Dockerfile)
 	if err != nil {
 		log.Error("Dockerfile stat", "Error", err)
 		return
 	}
 
 	haveBuilder := true
-	_, err = os.Stat(dockerfileBuild)
+	_, err = os.Stat(container.DockerfileBuild)
 	if err != nil {
 		haveBuilder = false
 	}
 
 	if haveBuilder {
 		var err error
+		var file *os.File
+
+		file, err = os.Open(container.DockerfileBuild)
+		if err != nil {
+			log.Error("os.Open", "path", container.DockerfileBuild, "Error", err)
+			return
+		}
+		defer file.Close()
+
+		builderBuildArgs := make([]string, 0)
+
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			match := buildArgRe.FindAllStringSubmatch(scanner.Text(), -1)
+			if len(match) == 1 && len(match[0]) == 2 {
+				log.Info("found build arg", "ARG", match[0][1])
+				builderBuildArgs = append(builderBuildArgs, match[0][1])
+			}
+		}
+		if err = scanner.Err(); err != nil {
+			log.Error("scanner", "Error", err)
+			return
+		}
+
+		builderContainerName := container.Name + "-builder"
 
-		buildBuilderCmd := exec.Command("docker", "build", "-t", containerName+"-builder", "-f", dockerfileBuild, ".")
+		buildBuilderCmdArgs := []string{
+			"build",
+			"-t", builderContainerName,
+			"-f", container.DockerfileBuild,
+		}
+		for _, arg := range builderBuildArgs {
+			if value, exists := os.LookupEnv(arg); exists {
+				buildBuilderCmdArgs = append(buildBuilderCmdArgs, "--build-arg", arg+"="+value)
+			} else if container.ValidateBuildArgsExist == nil || *container.ValidateBuildArgsExist == true {
+				log.Error("build arg " + arg + " missing from environment")
+				return
+			}
+		}
+		buildBuilderCmdArgs = append(buildBuilderCmdArgs, ".")
+		log.Debug("docker build", "Args", buildBuilderCmdArgs[1:])
+
+		defer exec.Command("docker", "rmi", "-f", builderContainerName).Run()
+
+		buildBuilderCmd := exec.Command("docker", buildBuilderCmdArgs...)
 		buildBuilderCmd.Stdout = os.Stdout
 		buildBuilderCmd.Stderr = os.Stderr
 		err = buildBuilderCmd.Run()
@@ -202,7 +249,7 @@ func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild
 			return
 		}
 
-		runCmd := exec.Command("docker", "run", "--rm", containerName+"-builder")
+		runCmd := exec.Command("docker", "run", "--rm", builderContainerName)
 
 		runCmdStdoutPipe, err := runCmd.StdoutPipe()
 		if err != nil {
@@ -211,8 +258,8 @@ func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild
 		}
 
 		buildCmd := exec.Command("docker", "build",
-			"-t", containerName,
-			"-f", dockerfile,
+			"-t", container.Name,
+			"-f", container.Dockerfile,
 			"-")
 		buildCmd.Stdin = runCmdStdoutPipe
 		buildCmd.Stdout = os.Stdout
@@ -226,16 +273,16 @@ func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild
 
 		err = buildCmd.Start()
 		if err != nil {
-			log.Error("build Dockerfile", "Error", err)
+			log.Error("docker build", "Error", err)
 			return
 		}
 
 		runCmd.Wait()
 		buildCmd.Wait()
 	} else {
 		buildCmd := exec.Command("docker", "build",
-			"-t", containerName,
-			"-f", dockerfile,
+			"-t", container.Name,
+			"-f", container.Dockerfile,
 			".")
 		buildCmd.Stdout = os.Stdout
 		buildCmd.Stderr = os.Stderr
@@ -258,7 +305,7 @@ func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild
 		dockerSaveLock.Lock()
 		defer dockerSaveLock.Unlock()
 
-		saveCmd := exec.Command("docker", "save", "-o", imagePath, containerName)
+		saveCmd := exec.Command("docker", "save", "-o", imagePath, container.Name)
 		saveCmd.Stdout = os.Stdout
 		saveCmd.Stderr = os.Stderr
 		err = saveCmd.Run()
@@ -271,7 +318,7 @@ func buildContainer(log log15.Logger, containerName, dockerfile, dockerfileBuild
 
 		log.Info("docker push")
 
-		pushCmd := exec.Command("docker", "push", containerName)
+		pushCmd := exec.Command("docker", "push", container.Name)
 		pushCmd.Stdout = os.Stdout
 		pushCmd.Stderr = os.Stderr
 		err := pushCmd.Run()