Merge pull request #686 from ystia/feature/GH-648-ansible-version

Feature/gh 648 ansible version

Author: laurentganne

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## UNRELEASED
 
+### DEPENDENCIES
+
+* The orchestrator requires now at least Ansible 2.10.0 (upgrade from 2.7.9 introduced in [GH-648](https://github.com/ystia/yorc/issues/648))
+
 ### FEATURES
 
 * Added an ElasticSearch store for events and logs ([GH-658](https://github.com/ystia/yorc/issues/658))
@@ -13,6 +17,7 @@
 
 ### ENHANCEMENTS
 
+* Upgrade Ansible version from 2.7.9 to 2.10.0 ([GH-648](https://github.com/ystia/yorc/issues/648))
 * Alien4Cloud download URL change ([GH-637](https://github.com/ystia/yorc/issues/637))
 * Enhance logs and events long-polling performances on file storage ([GH-654](https://github.com/ystia/yorc/issues/654))
 

commands/bootstrap/resources/topology/tosca_types.zip

@@ -422,6 +422,11 @@ func installDependencies(workingDirectoryPath string) error {
 		return err
 	}
 
+	// Install paramiko
+	if err := installParamiko(); err != nil {
+		return err
+	}
+
 	// Download Consul
 	overwrite := (previousBootstrapVersion.Consul != consulVersion)
 	if err := downloadUnzip(inputValues.Consul.DownloadURL, "consul.zip", workingDirectoryPath, overwrite); err != nil {
@@ -546,6 +551,22 @@ func installAnsible(version string) error {
 	return err
 }
 
+// installParamiko installs Paramiko needed when openSSH is not used
+// Install using pip3 if available else using pip
+func installParamiko() error {
+	pipCmd, err := getPipCmd()
+	if err != nil {
+		return err
+	}
+
+	// Installing ansible
+	cmd := exec.Command(pipCmd, "install", "--upgrade", "paramiko")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	return err
+}
+
 // getPipModuleInstalledVersion retruns the version of an installed pip module.
 // Returns an error if the module is not installed
 func getPipModuleInstalledVersion(pipCmd, module string) (string, error) {

commands/bootstrap/setup.go

@@ -58,7 +58,7 @@ It requires the following packages to be installed on the local host:
   * wget
 
 This basic installation on the local host will attempt to install without sudo privileges
-python ansible module |ansible_version| if needed as well as python packages
+python ansible module |ansible_version| if needed as well as paramiko and python packages
 MarkupSafe, jinja2, PyYAML, six, cryptography, setuptools.
 So if this ansible module or python packages are not yet installed on the local host,
 you could either add them yourself, with sudo privileges, running for example:
@@ -279,12 +279,12 @@ for example :
 
 The bootstrap configuration file can be also be used to define Ansible Inventory
 configuration parameters.
-This is needed for example if remote hosts have python3 installed by default and not python,
-like on Ubuntu 18+.
+This is needed for example if you want to use on target hosts a python version
+different from the one automatically selected by Ansible.
 
 In this case, you can add in the bootstrap configuration file, a section allowing
 to configure an Ansible behavioral inventory parameter that will allow to specify
-which python interpreter could be used by Ansible on remote hosts, as described in
+which python interpreter could be used by Ansible on target hosts, as described in
 :ref:`Ansible Inventory Configuration section <option_ansible_inventory_cfg>`.
 
 This would give for example in the bootstrap configuration file:
@@ -343,14 +343,12 @@ Example of a Google Cloud deployment configuration file
 
 .. _yorc_google_example_ubuntu_section:
 
-Example of a Google Cloud deployment configuration with Ubuntu 19.04 on-demand compute
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Example of a Google Cloud deployment configuration enforcing the use of python3
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-In this example, on-demand compute instances run Ubuntu 19.04 on which python3
-is installed by default (and not python).
-In this case, a specific Ansible behavioral inventory parameter
-``ansible_python_interpreter`` must be defined so that Ansible is able to find
-this python interpreter on the remote hosts.
+In this example, a specific Ansible behavioral inventory parameter
+``ansible_python_interpreter`` is defined so that Ansible will use the specified
+python interpreter on the target hosts.
 
 .. code-block:: YAML
 
@@ -375,11 +373,7 @@ this python interpreter on the remote hosts.
         project: myproject
   ansible:
     inventory:
-      # Remote host run Ubuntu 19.04, using python3.
-      # Defining here the Ansible behavioral inventory parameter ansible_python_interpreter
-      # pointing to python3.
-      # This is required or Ansible will attempt to use python on the remote host
-      # which will fail as python is not installed by default.
+      # Enforce the use of /usr/bin/python3 by Ansible on target hosts
       "target_hosts:vars":
       - ansible_python_interpreter=/usr/bin/python3
   address:

doc/bootstrap.rst

@@ -489,7 +489,7 @@ can be any parameter specific to your ansible playbooks, or `behavioral inventor
 parameters <https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#list-of-behavioral-inventory-parameters>`_
 describing how Ansible interacts with remote hosts.
 
-For example, for Ansible to use python3 on remote hosts, you must define
+For example, for Ansible to use a given python interpreter on target hosts, you must define
 the Ansible behavioral inventory parameter ``ansible_python_interpreter``
 in the Ansible inventory Yorc configuration, like below in Yaml:
 
@@ -503,16 +503,11 @@ in the Ansible inventory Yorc configuration, like below in Yaml:
 By default, the Orchestrator will define :
 
   * an inventory group ``target_hosts`` containing the list of remote hosts, and its
-    associated variable group ``target_hosts:vars`` configuring by default this
-    behavioral parameter:
+    associated variable group ``target_hosts:vars`` configuring by default these
+    behavioral parameters:
 
     * ``ansible_ssh_common_args="-o ConnectionAttempts=20"``
-
-  * an inventory group ``hosted_operations``  and its associated variable group ``hosted_operations:vars``
-    for operations that are executed on the orchestrator host, configuring by default
-    this behavioral parameter:
-
-    * ``ansible_python_interpreter=/usr/bin/env python``
+    * ``ansible_python_interpreter="auto_silent"``
 
 .. warning::
     Settings defined by the user take precedence over settings defined by the

doc/configuration.rst

@@ -63,6 +63,7 @@ Now you can proceed with the installation of softwares used by Yorc.
 .. parsed-literal::
 
     sudo pip install ansible==\ |ansible_version|
+    sudo pip install paramiko
     wget \https://releases.hashicorp.com/consul/\ |consul_version|\ /consul\_\ |consul_version|\ _linux_amd64.zip
     sudo unzip consul\_\ |consul_version|\ _linux_amd64.zip -d /usr/local/bin
     wget \https://releases.hashicorp.com/terraform/\ |terraform_version|\ /terraform\_\ |terraform_version|\ _linux_amd64.zip

doc/install.rst

@@ -215,14 +215,9 @@ In order to let Yorc interact with Docker to manage sandboxes some requirements
 Yorc uses standard Docker's APIs so ``DOCKER_HOST`` and ``DOCKER_CERT_PATH`` environment variables could be used
 to configure the way Yorc interacts with Docker.
 
-In order to execute operations on containers, either the following requirements
-should be met on Docker images used as sandboxes:
-
-  * the ``/usr/bin/env`` command should be present
-  * a python 2 interpreter compatible with ansible |ansible_version| should be available as the ``python`` command
-
-or Yorc configuration should provide a python interpreter path through the Ansible
-behavioral inventory parameter ``ansible_python_interpreter``, like below in a Yaml
+To execute operations on containers, Ansible will by default detect the python interpreter to use
+as described in `Ansible documentation <https://docs.ansible.com/ansible/latest/reference_appendices/interpreter_discovery.html>`_ section ``auto_silent``.
+To provide yourself a python interpreter path, use the Ansible behavioral inventory parameter ``ansible_python_interpreter``, like below in a Yaml
 Yorc configuration excerpt specifying to use python3 :
 
 .. code-block:: YAML

doc/tosca.rst

@@ -34,6 +34,22 @@ feature requires to have the ``management`` ACL. It is possible to disable this
 
 .. note:: A rolling upgrade without interruption feature is planned for future versions.
 
+.. _yorc_upgrades_410_section:
+
+Upgrading to Yorc 4.1.0
+-----------------------
+
+Ansible
+~~~~~~~
+
+Upgrade Ansible to version 2.10.0:
+
+.. code-block:: bash
+
+    sudo pip uninstall ansible
+    sudo pip install ansible==2.10.0
+    sudo pip install paramiko
+
 .. _yorc_upgrades_320_section:
 
 Upgrading to Yorc 3.2.0

doc/upgrade.rst

@@ -7,7 +7,7 @@ ARG TF_GOOGLE_PLUGIN_VERSION
 ARG TF_OPENSTACK_PLUGIN_VERSION
 # Update terraform default when possible
 ENV TERRAFORM_VERSION ${TERRAFORM_VERSION:-0.11.8}
-ENV ANSIBLE_VERSION ${ANSIBLE_VERSION:-2.7.9}
+ENV ANSIBLE_VERSION ${ANSIBLE_VERSION:-2.10.0}
 ENV TF_CONSUL_PLUGIN_VERSION ${TF_CONSUL_PLUGIN_VERSION:-2.1.0}
 ENV TF_AWS_PLUGIN_VERSION ${TF_AWS_PLUGIN_VERSION:-1.36.0}
 ENV TF_GOOGLE_PLUGIN_VERSION ${TF_GOOGLE_PLUGIN_VERSION:-1.18.0}

pkg/Dockerfile

@@ -64,7 +64,6 @@ const ansibleConfigDefaultsHeader = "defaults"
 const ansibleInventoryHostsHeader = "target_hosts"
 const ansibleInventoryHostedHeader = "hosted_operations"
 const ansibleInventoryHostsVarsHeader = ansibleInventoryHostsHeader + ":vars"
-const ansibleInventoryHostedVarsHeader = ansibleInventoryHostedHeader + ":vars"
 
 var ansibleDefaultConfig = map[string]map[string]string{
 	ansibleConfigDefaultsHeader: map[string]string{
@@ -83,9 +82,7 @@ var ansibleFactCaching = map[string]string{
 var ansibleInventoryConfig = map[string][]string{
 	ansibleInventoryHostsVarsHeader: []string{
 		"ansible_ssh_common_args=\"-o ConnectionAttempts=20\"",
-	},
-	ansibleInventoryHostedVarsHeader: []string{
-		"ansible_python_interpreter=/usr/bin/env python",
+		"ansible_python_interpreter=\"auto_silent\"",
 	},
 }
 

prov/ansible/execution.go

@@ -166,7 +166,7 @@ func endTaskSectionAndLogTaskResult(ansibleTask *ansibleTaskContext) {
 			// internal implementation doesn't have to appear in Consul logs.
 			// Logs for this script will be added in Consul.
 			// Logs of other internal tasks are skipped.
-			if ansibleTask.taskName == "command" {
+			if ansibleTask.taskName == "command" || ansibleTask.taskName == "shell" {
 				logScriptTaskResult(ansibleTask)
 			}
 		}

prov/ansible/yaml_output.go

@@ -138,7 +138,7 @@ resource "null_resource" "yorc-server-provisioning" {
       "sudo yum install -q -y python2-pip nfs-utils autofs",
       "cd /tmp && wget -q https://releases.hashicorp.com/terraform/0.9.11/terraform_0.9.11_linux_amd64.zip && sudo unzip /tmp/terraform_0.9.11_linux_amd64.zip -d /usr/local/bin",
       "sudo -H pip install -q pip --upgrade",
-      "sudo -H pip install -q ansible==2.7.9",
+      "sudo -H pip install -q ansible==2.10.0",
       "mv /tmp/config.yorc.json ~/config.yorc.json",
       "echo -e '/-    /etc/auto.direct\n' | sudo tee /etc/auto.master > /dev/null",
       "cat /tmp/auto.yorc | sudo tee /etc/auto.direct > /dev/null",

testdata/infra/openstack/yorc.tf

@@ -3,7 +3,7 @@ yorc_version: 4.1.0-SNAPSHOT
 alien4cloud_version: 2.2.0
 consul_version: 1.2.3
 terraform_version: 0.11.8
-ansible_version: 2.7.9
+ansible_version: 2.10.0
 tf_consul_plugin_version: 2.1.0
 tf_aws_plugin_version: 1.36.0
 tf_openstack_plugin_version: 1.9.0