Execute ansible in a sandbox container

config/config.go

@@ -84,6 +84,15 @@ const DefaultUpgradesConcurrencyLimit = 1000
 // DefaultSSHConnectionTimeout is the default timeout for SSH connections
 const DefaultSSHConnectionTimeout = 10 * time.Second
 
+// DefaultSandboxWorkDir is the default workdir in the sandbox container where we mount the ansible recipes
+const DefaultSandboxWorkDir = "/home/ansible"
+
+// DefaultSandboxOverlayDir is the default directory in the sandbox container where we mount the overlay
+const DefaultSandboxOverlayDir = "/home/overlay"
+
+// DefaultSandboxMountAgentSocket is the default location in the sandbox container where we mount the ssh agent socket
+const DefaultSandboxMountAgentSocket = "/ssh-agent"
+
 // Configuration holds config information filled by Cobra and Viper (see commands package for more information)
 type Configuration struct {
 	Ansible                          Ansible       `yaml:"ansible,omitempty" mapstructure:"ansible"`
@@ -120,6 +129,9 @@ type DockerSandbox struct {
 	Command    []string `mapstructure:"command"`
 	Entrypoint []string `mapstructure:"entrypoint"`
 	Env        []string `mapstructure:"env"`
+	User       string   `mapstructure:"user"`
+	Cpus       string   `mapstructure:"cpus"`
+	Memory     string   `mapstructure:"memory"`
 }
 
 // HostedOperations holds the configuration for operations executed on the orechestrator host (eg. with an operation_host equals to ORECHESTRATOR)

config/config_test.go

@@ -210,7 +210,7 @@ func TestHostedOperations_Format(t *testing.T) {
 	}{
 		{"DefaultValues", fields{}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:<nil>}`},
 		{"AllowUnsandboxed", fields{UnsandboxedOperationsAllowed: true}, `{UnsandboxedOperationsAllowed:true DefaultSandbox:<nil>}`},
-		{"DefaultSandboxConfigured", fields{DefaultSandbox: &DockerSandbox{Image: "alpine:3.7", Command: []string{"cmd", "arg"}}}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:&{Image:alpine:3.7 Command:[cmd arg] Entrypoint:[] Env:[]}}`},
+		{"DefaultSandboxConfigured", fields{DefaultSandbox: &DockerSandbox{Image: "alpine:3.7", Command: []string{"cmd", "arg"}, User: "1000:1000", Cpus: "1", Memory: "200m"}}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:&{Image:alpine:3.7 Command:[cmd arg] Entrypoint:[] Env:[] User:1000:1000 Cpus:1 Memory:200m}}`},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/armon/go-metrics v0.3.0
 	github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
@@ -109,6 +110,7 @@ require (
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/tools v0.0.0-20200302225559-9b52d559c609
 	gopkg.in/AlecAivazis/survey.v1 v1.6.3
 	gopkg.in/cookieo9/resources-go.v2 v2.0.0-20150225115733-d27c04069d0d
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect

helper/sshutil/sshutil.go

@@ -17,6 +17,7 @@ package sshutil
 import (
 	"bytes"
 	"fmt"
+	uuid "github.com/satori/go.uuid"
 	"io"
 	"io/ioutil"
 	"net"
@@ -268,14 +269,23 @@ func (client *SSHClient) CopyFile(source io.Reader, remotePath string, permissio
 	return nil
 }
 
-// NewSSHAgent allows to return a new SSH Agent
-func NewSSHAgent(ctx context.Context) (*SSHAgent, error) {
+// NewSSHAgent allows to return a new SSH Agent. If socketDir is specified, create the agent socket at /socketDir/<uuid>/agent
+func NewSSHAgent(ctx context.Context, socketDir string) (*SSHAgent, error) {
 	bin, err := exec.LookPath("ssh-agent")
 	if err != nil {
 		return nil, errors.Wrap(err, "could not find ssh-agent")
 	}
 
 	cmd := executil.Command(ctx, bin)
+	if socketDir != "" {
+		tempDir := filepath.Join(socketDir, uuid.NewV4().String())
+		if err = os.MkdirAll(tempDir, 0770); err != nil {
+			return nil, errors.Wrapf(err, "could not create socket directory for ssh-agent at %q", tempDir)
+		}
+		bindAddress := filepath.Join(tempDir, "agent")
+		log.Debugf("Set ssh-agent bind-address to %q", bindAddress)
+		cmd.Args = append(cmd.Args, "-a", bindAddress)
+	}
 	out, err := cmd.Output()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to run ssh-agent")

helper/sshutil/sshutil_test.go

@@ -20,6 +20,8 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
+	"github.com/stretchr/testify/assert"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -39,7 +41,7 @@ func TestSSHAgent(t *testing.T) {
 	privateKeyContent := string(bArray)
 
 	// Create new ssh-agent
-	sshAg, err := NewSSHAgent(context.Background())
+	sshAg, err := NewSSHAgent(context.Background(), "")
 	require.Nil(t, err, "unexpected error while creating SSH-agent")
 	defer func() {
 		err = sshAg.Stop()
@@ -67,6 +69,25 @@ func TestSSHAgent(t *testing.T) {
 	keys, err = sshAg.agent.List()
 	require.Nil(t, err)
 	require.Len(t, keys, 0, "no key expected")
+
+	// test create more than one agents in a specific socket directory
+	sshAg1, err := NewSSHAgent(context.Background(), "/tmp/ssh-agent")
+	require.Nil(t, err)
+	sshAg2, err := NewSSHAgent(context.Background(), "/tmp/ssh-agent")
+	require.Nil(t, err)
+	require.DirExists(t, "/tmp/ssh-agent")
+	assert.NotEqual(t, sshAg1.Socket, sshAg2.Socket)
+	// stop one ssh-agent does not remove the parent socket directory but the given agent socket only
+	err = sshAg1.Stop()
+	require.Nil(t, err)
+	require.DirExists(t, "/tmp/ssh-agent", "stop one ssh-agent removed the parent socket directory")
+	if _, err := os.Stat(sshAg1.Socket); err == nil {
+		t.Errorf("failed to remove agent socket when stopping ssh-agent %v", err)
+	}
+	require.FileExists(t, sshAg2.Socket, "stop one ssh-agent but removed another agent socket")
+	err = sshAg2.Stop()
+	err = os.RemoveAll("/tmp/ssh-agent")
+	require.Nil(t, err, "failed to clean up /tmp/ssh-agent after test")
 }
 
 // BER SSH key is not handled by crypto/ssh

pkg/ansible/Dockerfile

@@ -0,0 +1,15 @@
+# Dockerfile for building the sandbox container
+
+FROM alpine:3.10
+
+ENV ANSIBLE_VERSION=2.9.9
+
+RUN apk add --no-cache --progress bash python3 openssl ca-certificates openssh sshpass && \
+    addgroup -g 1000 ansible && \
+    adduser -D -s /bin/bash -g ansible -G ansible -u 1000 ansible && \
+    if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \
+	apk --update add --virtual build-dependencies python3-dev libffi-dev openssl-dev build-base && \
+    pip3 install --upgrade pip && \
+	pip3 install ansible==${ANSIBLE_VERSION} && \
+	apk del build-dependencies && \
+	rm -rf /var/cache/apk/*

prov/ansible/consul_test.go

@@ -43,4 +43,7 @@ func TestRunConsulAnsiblePackageTests(t *testing.T) {
 	t.Run("TestLogAnsibleOutputInConsulFromScriptFailure", func(t *testing.T) {
 		testLogAnsibleOutputInConsulFromScriptFailure(t)
 	})
+	t.Run("TestSandbox", func(t *testing.T) {
+		testCreatesandbox(t)
+	})
 }

prov/ansible/execution.go

@@ -61,6 +61,7 @@ print(os.environ['VAULT_PASSWORD'])
 `
 
 const ansibleConfigDefaultsHeader = "defaults"
+const ansibleConfigSSHConnection = "ssh_connection"
 const ansibleInventoryHostsHeader = "target_hosts"
 const ansibleInventoryHostedHeader = "hosted_operations"
 const ansibleInventoryHostsVarsHeader = ansibleInventoryHostsHeader + ":vars"
@@ -73,6 +74,7 @@ var ansibleDefaultConfig = map[string]map[string]string{
 		"stdout_callback":   "yaml",
 		"nocows":            "1",
 	},
+	ansibleConfigSSHConnection: map[string]string{},
 }
 
 var ansibleFactCaching = map[string]string{
@@ -139,7 +141,7 @@ type execution interface {
 }
 
 type ansibleRunner interface {
-	runAnsible(ctx context.Context, retry bool, currentInstance, ansibleRecipePath string) error
+	generateRunAnsible(ctx context.Context, currentInstance, ansibleRecipePath string) (handler outputHandler, err error)
 }
 
 type executionCommon struct {
@@ -788,26 +790,13 @@ func (e *executionCommon) execute(ctx context.Context, retry bool) error {
 }
 
 func (e *executionCommon) generateHostConnectionForOrchestratorOperation(ctx context.Context, buffer *bytes.Buffer) error {
-	if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
-		var err error
-		e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID)
-		if err != nil {
-			return err
-		}
-		buffer.WriteString(" ansible_connection=docker ansible_host=")
-		buffer.WriteString(e.containerID)
-	} else if e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed {
-		buffer.WriteString(" ansible_connection=local")
-	} else {
-		actualRootCause := "there is no sandbox configured to handle it"
-		if e.cli == nil {
-			actualRootCause = "connection to docker failed (see logs)"
-		}
-
-		err := errors.Errorf("Ansible provisioning: you are trying to execute an operation on the orchestrator host but %s and execution on the actual orchestrator host is disallowed by configuration", actualRootCause)
+	if e.cfg.Ansible.HostedOperations.DefaultSandbox == nil && !e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed {
+		err := errors.Errorf("Ansible provisioning: you are trying to execute an operation on the orchestrator host " +
+			"but there is no sandbox configured to handle it and execution on the actual orchestrator host is disallowed by configuration")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).Registerf("%v", err)
 		return err
 	}
+	buffer.WriteString(" ansible_connection=local")
 	return nil
 }
 
@@ -935,7 +924,7 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry
 	}
 
 	vaultPassScript := fmt.Sprintf(vaultPassScriptFormat, pythonInterpreter)
-	if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, ".vault_pass"), []byte(vaultPassScript), 0764); err != nil {
+	if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, ".vault_pass"), []byte(vaultPassScript), 0750); err != nil {
 		err = errors.Wrap(err, "Failed to write .vault_pass file")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
 		return err
@@ -1087,11 +1076,13 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry
 		tarPath := filepath.Join(ansibleRecipePath, artifactName+".tar")
 		buildArchive(e.OverlayPath, artifactPath, tarPath)
 	}
-
-	err = e.ansibleRunner.runAnsible(ctx, retry, currentInstance, ansibleRecipePath)
+	outputHandler, err := e.ansibleRunner.generateRunAnsible(ctx, currentInstance, ansibleRecipePath)
 	if err != nil {
 		return err
 	}
+	if err = e.executePlaybook(ctx, retry, ansibleRecipePath, outputHandler); err != nil {
+		return err
+	}
 	if e.HaveOutput {
 		outputsFiles, err := filepath.Glob(filepath.Join(ansibleRecipePath, "*-out.csv"))
 		if err != nil {
@@ -1208,9 +1199,55 @@ func (e *executionCommon) getInstanceIDFromHost(host string) (string, error) {
 
 func (e *executionCommon) executePlaybook(ctx context.Context, retry bool,
 	ansibleRecipePath string, handler outputHandler) error {
-	cmd := executil.Command(ctx, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", filepath.Join(ansibleRecipePath, ".vault_pass"))
 	env := os.Environ()
 	env = append(env, "VAULT_PASSWORD="+e.vaultToken)
+	var sshAgentSocket string
+	if !e.cfg.DisableSSHAgent {
+		socketDir, err := filepath.Abs(filepath.Join(e.cfg.WorkingDirectory, "ssh-agent"))
+		if err != nil {
+			return err
+		}
+		// Check if SSHAgent is needed
+		sshAgent, err := e.configureSSHAgent(ctx, socketDir)
+		if err != nil {
+			return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution")
+		}
+		if sshAgent != nil {
+			sshAgentSocket = sshAgent.Socket
+			log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent")
+			if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+				env = append(env, "SSH_AUTH_SOCK="+config.DefaultSandboxMountAgentSocket)
+			} else {
+				env = append(env, "SSH_AUTH_SOCK="+sshAgentSocket)
+			}
+			defer func() {
+				err = sshAgent.RemoveAllKeys()
+				if err != nil {
+					log.Debugf("Warning: failed to remove all SSH agents keys due to error:%+v", err)
+				}
+				err = sshAgent.Stop()
+				if err != nil {
+					log.Debugf("Warning: failed to stop SSH agent due to error:%+v", err)
+				}
+			}()
+		}
+	}
+	var cmd *executil.Cmd
+	if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+		log.Debugf("Start sandbox container with mount directories ansibleRecipePath: %s, overlayPath: %s", ansibleRecipePath, e.OverlayPath)
+		var err error
+		if e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID,
+			ansibleRecipePath, e.OverlayPath, sshAgentSocket, env); err != nil {
+			return err
+		}
+		log.Debugf("Execute ansible-playbook in sandbox container: %s", e.containerID)
+		cmd = executil.Command(ctx, "docker", "exec", "--workdir", config.DefaultSandboxWorkDir, e.containerID, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", ".vault_pass")
+	} else {
+		log.Debugf("Execute ansible-playbook on localhost")
+		cmd = executil.Command(ctx, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", filepath.Join(ansibleRecipePath, ".vault_pass"))
+		cmd.Env = env
+	}
+
 	if _, err := os.Stat(filepath.Join(ansibleRecipePath, "run.ansible.retry")); retry && (err == nil || !os.IsNotExist(err)) {
 		cmd.Args = append(cmd.Args, "--limit", filepath.Join("@", ansibleRecipePath, "run.ansible.retry"))
 	}
@@ -1228,31 +1265,8 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool,
 		} else {
 			cmd.Args = append(cmd.Args, "-c", "paramiko")
 		}
-
-		if !e.cfg.DisableSSHAgent {
-			// Check if SSHAgent is needed
-			sshAgent, err := e.configureSSHAgent(ctx)
-			if err != nil {
-				return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution")
-			}
-			if sshAgent != nil {
-				log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent")
-				env = append(env, "SSH_AUTH_SOCK="+sshAgent.Socket)
-				defer func() {
-					err = sshAgent.RemoveAllKeys()
-					if err != nil {
-						log.Debugf("Warning: failed to remove all SSH agents keys due to error:%+v", err)
-					}
-					err = sshAgent.Stop()
-					if err != nil {
-						log.Debugf("Warning: failed to stop SSH agent due to error:%+v", err)
-					}
-				}()
-			}
-		}
 	}
 	cmd.Dir = ansibleRecipePath
-	cmd.Env = env
 	errbuf := events.NewBufferedLogEntryWriter()
 	cmd.Stderr = errbuf
 
@@ -1277,7 +1291,7 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool,
 	return nil
 }
 
-func (e *executionCommon) configureSSHAgent(ctx context.Context) (*sshutil.SSHAgent, error) {
+func (e *executionCommon) configureSSHAgent(ctx context.Context, socketDir string) (*sshutil.SSHAgent, error) {
 	var addSSHAgent bool
 	for _, host := range e.hosts {
 		if len(host.privateKeys) > 0 {
@@ -1289,7 +1303,7 @@ func (e *executionCommon) configureSSHAgent(ctx context.Context) (*sshutil.SSHAg
 		return nil, nil
 	}
 
-	agent, err := sshutil.NewSSHAgent(ctx)
+	agent, err := sshutil.NewSSHAgent(ctx, socketDir)
 	if err != nil {
 		return nil, err
 	}
@@ -1400,12 +1414,32 @@ func (e *executionCommon) generateAnsibleConfigurationFile(
 
 	ansibleConfig := getAnsibleConfigFromDefault()
 
-	// Adding settings whose values are known at runtime, related to the deployment
-	// directory path
-	ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = ansibleRecipePath
-	if e.CacheFacts {
-		ansibleFactCaching["fact_caching_connection"] = path.Join(ansiblePath, "facts_cache")
+	// Adding settings whose values are known at runtime, related to the deployment directory path
+	if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+		ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = config.DefaultSandboxWorkDir
+		// Specify where ansible can write its temp files in the sandbox container.
+		// (By default, ansible writes to ~/.ansible/tmp on host machine and may cause permission denied from inside the container)
+		ansibleConfig[ansibleConfigDefaultsHeader]["local_tmp"] = path.Join(config.DefaultSandboxWorkDir, "tmp")
+		// Specify where ansible writes SSH control path sockets in the sandbox container for SSH multiplexing.
+		// (By default, ansible writes to ~/.ansible/cp on host machine and may cause permission denied from inside the container)
+		// Each sandbox container should have its own control path
+		if e.cfg.Ansible.UseOpenSSH {
+			ansibleConfig[ansibleConfigSSHConnection]["control_path_dir"] = path.Join(config.DefaultSandboxWorkDir, "cp")
+		}
+	} else {
+		ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = ansibleRecipePath
+	}
 
+	if e.CacheFacts {
+		if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+			// location in sandbox to save ansible gathering facts
+			// each yorc task should have its own facts (i.e., do not share the gathering facts from different tasks
+			// even in the same deploymentID) so that one task do not overwrite the facts of the other ones unexpectedly
+			// (e.g., if share facts, set "become: true" in one task overwrites the USER fact from ubuntu to root).
+			ansibleFactCaching["fact_caching_connection"] = path.Join(config.DefaultSandboxWorkDir, "facts_cache")
+		} else {
+			ansibleFactCaching["fact_caching_connection"] = path.Join(ansibleRecipePath, "facts_cache")
+		}
 		for k, v := range ansibleFactCaching {
 			ansibleConfig[ansibleConfigDefaultsHeader][k] = v
 		}