Execute ansible in a sandbox container

config/config.go

@@ -84,6 +84,15 @@ const DefaultUpgradesConcurrencyLimit = 1000
 // DefaultSSHConnectionTimeout is the default timeout for SSH connections
 const DefaultSSHConnectionTimeout = 10 * time.Second
 
+// DefaultSandboxWorkDir is the default workdir in the sandbox container where we mount the ansible recipes
+const DefaultSandboxWorkDir = "/home/ansible"
+
+// DefaultSandboxOverlayDir is the default directory in the sandbox container where we mount the overlay
+const DefaultSandboxOverlayDir = "/home/overlay"
+
+// DefaultSandboxMountAgentSocket is the default location in the sandbox container where we mount the ssh agent socket
+const DefaultSandboxMountAgentSocket = "/ssh-agent"
+
 // Configuration holds config information filled by Cobra and Viper (see commands package for more information)
 type Configuration struct {
 	Ansible                          Ansible       `yaml:"ansible,omitempty" mapstructure:"ansible"`
@@ -120,6 +129,9 @@ type DockerSandbox struct {
 	Command    []string `mapstructure:"command"`
 	Entrypoint []string `mapstructure:"entrypoint"`
 	Env        []string `mapstructure:"env"`
+	User       string   `mapstructure:"user"`
+	Cpus       string   `mapstructure:"cpus"`
+	Memory     string   `mapstructure:"memory"`
 }
 
 // HostedOperations holds the configuration for operations executed on the orechestrator host (eg. with an operation_host equals to ORECHESTRATOR)

config/config_test.go

@@ -210,7 +210,7 @@ func TestHostedOperations_Format(t *testing.T) {
 	}{
 		{"DefaultValues", fields{}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:<nil>}`},
 		{"AllowUnsandboxed", fields{UnsandboxedOperationsAllowed: true}, `{UnsandboxedOperationsAllowed:true DefaultSandbox:<nil>}`},
-		{"DefaultSandboxConfigured", fields{DefaultSandbox: &DockerSandbox{Image: "alpine:3.7", Command: []string{"cmd", "arg"}}}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:&{Image:alpine:3.7 Command:[cmd arg] Entrypoint:[] Env:[]}}`},
+		{"DefaultSandboxConfigured", fields{DefaultSandbox: &DockerSandbox{Image: "alpine:3.7", Command: []string{"cmd", "arg"}, User: "1000:1000", Cpus: "1", Memory: "200m"}}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:&{Image:alpine:3.7 Command:[cmd arg] Entrypoint:[] Env:[] User:1000:1000 Cpus:1 Memory:200m}}`},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/ansible/Dockerfile

@@ -0,0 +1,15 @@
+# Dockerfile for building the sandbox container
+
+FROM alpine:3.10
+
+ENV ANSIBLE_VERSION=2.9.9
+
+RUN apk add --no-cache --progress bash python3 openssl ca-certificates openssh sshpass && \
+    addgroup -g 1000 ansible && \
+    adduser -D -s /bin/bash -g ansible -G ansible -u 1000 ansible && \
+    if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \
+	apk --update add --virtual build-dependencies python3-dev libffi-dev openssl-dev build-base && \
+    pip3 install --upgrade pip && \
+	pip3 install ansible==${ANSIBLE_VERSION} && \
+	apk del build-dependencies && \
+	rm -rf /var/cache/apk/*

prov/ansible/consul_test.go

@@ -43,4 +43,7 @@ func TestRunConsulAnsiblePackageTests(t *testing.T) {
 	t.Run("TestLogAnsibleOutputInConsulFromScriptFailure", func(t *testing.T) {
 		testLogAnsibleOutputInConsulFromScriptFailure(t)
 	})
+	t.Run("TestSandbox", func(t *testing.T) {
+		testCreatesandbox(t)
+	})
 }

prov/ansible/execution.go

@@ -61,6 +61,7 @@ print(os.environ['VAULT_PASSWORD'])
 `
 
 const ansibleConfigDefaultsHeader = "defaults"
+const ansibleConfigSSHConnection = "ssh_connection"
 const ansibleInventoryHostsHeader = "target_hosts"
 const ansibleInventoryHostedHeader = "hosted_operations"
 const ansibleInventoryHostsVarsHeader = ansibleInventoryHostsHeader + ":vars"
@@ -73,6 +74,7 @@ var ansibleDefaultConfig = map[string]map[string]string{
 		"stdout_callback":   "yaml",
 		"nocows":            "1",
 	},
+	ansibleConfigSSHConnection: map[string]string{},
 }
 
 var ansibleFactCaching = map[string]string{
@@ -139,7 +141,7 @@ type execution interface {
 }
 
 type ansibleRunner interface {
-	runAnsible(ctx context.Context, retry bool, currentInstance, ansibleRecipePath string) error
+	generateRunAnsible(ctx context.Context, currentInstance, ansibleRecipePath string) (handler outputHandler, err error)
 }
 
 type executionCommon struct {
@@ -788,26 +790,13 @@ func (e *executionCommon) execute(ctx context.Context, retry bool) error {
 }
 
 func (e *executionCommon) generateHostConnectionForOrchestratorOperation(ctx context.Context, buffer *bytes.Buffer) error {
-	if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
-		var err error
-		e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID)
-		if err != nil {
-			return err
-		}
-		buffer.WriteString(" ansible_connection=docker ansible_host=")
-		buffer.WriteString(e.containerID)
-	} else if e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed {
-		buffer.WriteString(" ansible_connection=local")
-	} else {
-		actualRootCause := "there is no sandbox configured to handle it"
-		if e.cli == nil {
-			actualRootCause = "connection to docker failed (see logs)"
-		}
-
-		err := errors.Errorf("Ansible provisioning: you are trying to execute an operation on the orchestrator host but %s and execution on the actual orchestrator host is disallowed by configuration", actualRootCause)
+	if e.cfg.Ansible.HostedOperations.DefaultSandbox == nil && !e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed {
+		err := errors.Errorf("Ansible provisioning: you are trying to execute an operation on the orchestrator host " +
+			"but there is no sandbox configured to handle it and execution on the actual orchestrator host is disallowed by configuration")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).Registerf("%v", err)
 		return err
 	}
+	buffer.WriteString(" ansible_connection=local")
 	return nil
 }
 
@@ -1087,11 +1076,13 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry
 		tarPath := filepath.Join(ansibleRecipePath, artifactName+".tar")
 		buildArchive(e.OverlayPath, artifactPath, tarPath)
 	}
-
-	err = e.ansibleRunner.runAnsible(ctx, retry, currentInstance, ansibleRecipePath)
+	outputHandler, err := e.ansibleRunner.generateRunAnsible(ctx, currentInstance, ansibleRecipePath)
 	if err != nil {
 		return err
 	}
+	if err = e.executePlaybook(ctx, retry, ansibleRecipePath, outputHandler); err != nil {
+		return err
+	}
 	if e.HaveOutput {
 		outputsFiles, err := filepath.Glob(filepath.Join(ansibleRecipePath, "*-out.csv"))
 		if err != nil {
@@ -1208,9 +1199,51 @@ func (e *executionCommon) getInstanceIDFromHost(host string) (string, error) {
 
 func (e *executionCommon) executePlaybook(ctx context.Context, retry bool,
 	ansibleRecipePath string, handler outputHandler) error {
-	cmd := executil.Command(ctx, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", filepath.Join(ansibleRecipePath, ".vault_pass"))
 	env := os.Environ()
 	env = append(env, "VAULT_PASSWORD="+e.vaultToken)
+	var sshAgentSocket string
+	if !e.cfg.DisableSSHAgent {
+		// Check if SSHAgent is needed
+		sshAgent, err := e.configureSSHAgent(ctx)
+		if err != nil {
+			return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution")
+		}
+		if sshAgent != nil {
+			sshAgentSocket = sshAgent.Socket
+			log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent")
+			if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+				env = append(env, "SSH_AUTH_SOCK="+config.DefaultSandboxMountAgentSocket)
+			} else {
+				env = append(env, "SSH_AUTH_SOCK="+sshAgentSocket)
+			}
+			defer func() {
+				err = sshAgent.RemoveAllKeys()
+				if err != nil {
+					log.Debugf("Warning: failed to remove all SSH agents keys due to error:%+v", err)
+				}
+				err = sshAgent.Stop()
+				if err != nil {
+					log.Debugf("Warning: failed to stop SSH agent due to error:%+v", err)
+				}
+			}()
+		}
+	}
+	var cmd *executil.Cmd
+	if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+		log.Debugf("Start sandbox container with mount directories ansibleRecipePath: %s, overlayPath: %s", ansibleRecipePath, e.OverlayPath)
+		var err error
+		if e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID,
+			ansibleRecipePath, e.OverlayPath, sshAgentSocket, env); err != nil {
+			return err
+		}
+		log.Debugf("Execute ansible-playbook in sandbox container: %s", e.containerID)
+		cmd = executil.Command(ctx, "docker", "exec", "--workdir", config.DefaultSandboxWorkDir, e.containerID, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", ".vault_pass")
+	} else {
+		log.Debugf("Execute ansible-playbook on localhost")
+		cmd = executil.Command(ctx, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", filepath.Join(ansibleRecipePath, ".vault_pass"))
+		cmd.Env = env
+	}
+
 	if _, err := os.Stat(filepath.Join(ansibleRecipePath, "run.ansible.retry")); retry && (err == nil || !os.IsNotExist(err)) {
 		cmd.Args = append(cmd.Args, "--limit", filepath.Join("@", ansibleRecipePath, "run.ansible.retry"))
 	}
@@ -1228,31 +1261,8 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool,
 		} else {
 			cmd.Args = append(cmd.Args, "-c", "paramiko")
 		}
-
-		if !e.cfg.DisableSSHAgent {
-			// Check if SSHAgent is needed
-			sshAgent, err := e.configureSSHAgent(ctx)
-			if err != nil {
-				return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution")
-			}
-			if sshAgent != nil {
-				log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent")
-				env = append(env, "SSH_AUTH_SOCK="+sshAgent.Socket)
-				defer func() {
-					err = sshAgent.RemoveAllKeys()
-					if err != nil {
-						log.Debugf("Warning: failed to remove all SSH agents keys due to error:%+v", err)
-					}
-					err = sshAgent.Stop()
-					if err != nil {
-						log.Debugf("Warning: failed to stop SSH agent due to error:%+v", err)
-					}
-				}()
-			}
-		}
 	}
 	cmd.Dir = ansibleRecipePath
-	cmd.Env = env
 	errbuf := events.NewBufferedLogEntryWriter()
 	cmd.Stderr = errbuf
 
@@ -1400,12 +1410,32 @@ func (e *executionCommon) generateAnsibleConfigurationFile(
 
 	ansibleConfig := getAnsibleConfigFromDefault()
 
-	// Adding settings whose values are known at runtime, related to the deployment
-	// directory path
-	ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = ansibleRecipePath
-	if e.CacheFacts {
-		ansibleFactCaching["fact_caching_connection"] = path.Join(ansiblePath, "facts_cache")
+	// Adding settings whose values are known at runtime, related to the deployment directory path
+	if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+		ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = config.DefaultSandboxWorkDir
+		// Specify where ansible can write its temp files in the sandbox container.
+		// (By default, ansible writes to ~/.ansible/tmp on host machine and may cause permission denied from inside the container)
+		ansibleConfig[ansibleConfigDefaultsHeader]["local_tmp"] = path.Join(config.DefaultSandboxWorkDir, "tmp")
+		// Specify where ansible writes SSH control path sockets in the sandbox container for SSH multiplexing.
+		// (By default, ansible writes to ~/.ansible/cp on host machine and may cause permission denied from inside the container)
+		// Each sandbox contrainer should have its own control path
+		if e.cfg.Ansible.UseOpenSSH {
+			ansibleConfig[ansibleConfigSSHConnection]["control_path_dir"] = path.Join(config.DefaultSandboxWorkDir, "cp")
+		}
+	} else {
+		ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = ansibleRecipePath
+	}
 
+	if e.CacheFacts {
+		if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+			// location in sandbox to save ansible gathering facts
+			// each yorc task should have its own facts (i.e., do not share the gathering facts from different tasks
+			// even in the same deploymentID) so that one task do not overwrite the facts of the other ones unexpectedly
+			// (e.g., if share facts, set "become: true" in one task overwrites the USER fact from ubuntu to root).
+			ansibleFactCaching["fact_caching_connection"] = path.Join(config.DefaultSandboxWorkDir, "facts_cache")
+		} else {
+			ansibleFactCaching["fact_caching_connection"] = path.Join(ansibleRecipePath, "facts_cache")
+		}
 		for k, v := range ansibleFactCaching {
 			ansibleConfig[ansibleConfigDefaultsHeader][k] = v
 		}

prov/ansible/execution_ansible.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/ystia/yorc/v4/config"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -65,10 +66,24 @@ type executionAnsible struct {
 	isAlienAnsible bool
 }
 
-func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentInstance, ansibleRecipePath string) error {
+func (e *executionAnsible) generateRunAnsible(ctx context.Context, currentInstance, ansibleRecipePath string) (outputHandler, error) {
 	var err error
+	outputHandler := &playbookOutputHandler{}
+	// for operation on host machine, set the ansible destination folder to the ansible recipe path on host machine
+	// for operation on sandbox, set the ansible destination folder and overlay to the default mount path inside the container
+	var overlayPathOnHost, destFolder string
+	if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+		overlayPathOnHost = e.OverlayPath
+		e.OverlayPath = config.DefaultSandboxOverlayDir
+		destFolder = config.DefaultSandboxWorkDir
+		defer func() {
+			e.OverlayPath = overlayPathOnHost
+		}()
+	} else {
+		destFolder = ansibleRecipePath
+	}
 	if !e.isAlienAnsible {
-		e.PlaybookPath, err = filepath.Abs(filepath.Join(e.OverlayPath, e.Primary))
+		e.PlaybookPath = filepath.Join(e.OverlayPath, e.Primary)
 	} else {
 		var playbook string
 		for _, envInput := range e.EnvInputs {
@@ -81,19 +96,19 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn
 			err = errors.New("No PLAYBOOK_ENTRY input found for an alien4cloud ansible implementation")
 		}
 		if err == nil {
-			e.PlaybookPath, err = filepath.Abs(filepath.Join(e.OverlayPath, filepath.Dir(e.Primary), playbook))
+			e.PlaybookPath = filepath.Join(e.OverlayPath, filepath.Dir(e.Primary), playbook)
 		}
 	}
 	if err != nil {
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-		return err
+		return outputHandler, err
 	}
 
 	ansibleGroupsVarsPath := filepath.Join(ansibleRecipePath, "group_vars")
 	if err = os.MkdirAll(ansibleGroupsVarsPath, 0775); err != nil {
 		err = errors.Wrap(err, "Failed to create group_vars directory: ")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-		return err
+		return outputHandler, err
 	}
 	var buffer bytes.Buffer
 	for _, envInput := range e.EnvInputs {
@@ -103,7 +118,7 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn
 		}
 		v, err := e.encodeEnvInputValue(envInput, ansibleRecipePath)
 		if err != nil {
-			return err
+			return outputHandler, err
 		}
 		buffer.WriteString(fmt.Sprintf("%s: %s", envInput.Name, v))
 		buffer.WriteString("\n")
@@ -135,19 +150,19 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn
 	for contextKey, contextValue := range e.CapabilitiesCtx {
 		v, err := e.encodeTOSCAValue(contextValue, ansibleRecipePath)
 		if err != nil {
-			return err
+			return outputHandler, err
 		}
 		buffer.WriteString(fmt.Sprintf("%s: %s", contextKey, v))
 		buffer.WriteString("\n")
 	}
 	buffer.WriteString("dest_folder: \"")
-	buffer.WriteString(ansibleRecipePath)
+	buffer.WriteString(destFolder)
 	buffer.WriteString("\"\n")
 
 	if err = ioutil.WriteFile(filepath.Join(ansibleGroupsVarsPath, "all.yml"), buffer.Bytes(), 0664); err != nil {
 		err = errors.Wrap(err, "Failed to write global group vars file: ")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-		return err
+		return outputHandler, err
 	}
 
 	if e.HaveOutput {
@@ -162,7 +177,7 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn
 		if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, "outputs.csv.j2"), buffer.Bytes(), 0664); err != nil {
 			err = errors.Wrap(err, "Failed to generate operation outputs file: ")
 			events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-			return err
+			return outputHandler, err
 		}
 	}
 
@@ -184,21 +199,22 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn
 	if err != nil {
 		err = errors.Wrap(err, "Failed to generate ansible playbook")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-		return err
+		return outputHandler, err
 	}
 	if err = tmpl.Execute(&buffer, e); err != nil {
 		err = errors.Wrap(err, "Failed to Generate ansible playbook template")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-		return err
+		return outputHandler, err
 	}
 	if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, "run.ansible.yml"), buffer.Bytes(), 0664); err != nil {
 		err = errors.Wrap(err, "Failed to write playbook file")
 		events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error())
-		return err
+		return outputHandler, err
+	}
+	if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil {
+		e.OverlayPath = overlayPathOnHost
 	}
-
 	events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, e.deploymentID).RegisterAsString(fmt.Sprintf("Ansible recipe for node %q: executing %q on remote host(s)", e.NodeName, filepath.Base(e.PlaybookPath)))
 
-	outputHandler := &playbookOutputHandler{execution: e, context: ctx}
-	return e.executePlaybook(ctx, retry, ansibleRecipePath, outputHandler)
+	return &playbookOutputHandler{execution: e, context: ctx}, err
 }