Qesap log ansible filename (#281)

* Reorganize order of ansible cmd stages

Reorganize the internal implementation of how the list of ansible
command is composed by the glue script. This commit does not have any
behavioral changes.

* Improve ansible log name

Improve the way the log filename is calculated from the ansible playbook
name. Name is no more detected within the command at fixed position but
by parsing each element of the command and detecting playbook filename
by the path. This new approach allow further change in the
ansible-playbook command composition. This commit also restrict the dump
to file only at ansible-playbook execution, leaving out the first
ansible command.

scripts/qesap/lib/cmds.py

@@ -245,8 +245,22 @@ def ansible_command_sequence(configure_data_ansible, base_project, sequence, ver
         list of list of strings, each command is rappresented as a list of its arguments
     """
 
-    # if the config.yaml has playbooks, the ansible and ansible-playbooks executables
-    # has to be available too
+    # 1. Create the environment variable set
+    #    that will be used by any command
+    original_env = dict(os.environ)
+    original_env['ANSIBLE_PIPELINING'] = 'True'
+    ansible_callbacks = []
+    if profile:
+        ansible_callbacks.append('ansible.posix.profile_tasks')
+    if junit:
+        ansible_callbacks.append('junit')
+        original_env['JUNIT_OUTPUT_DIR'] = junit
+    if len(ansible_callbacks) > 0:
+        original_env['ANSIBLE_CALLBACKS_ENABLED'] = ','.join(ansible_callbacks)
+    if 'roles_path' in configure_data_ansible:
+        original_env['ANSIBLE_ROLES_PATH'] = configure_data_ansible['roles_path']
+
+    # 2. Verify that needed binary are usable
     ansible_bin_paths = {}
     for ansible_bin in ['ansible', 'ansible-playbook']:
         binpath = shutil.which(ansible_bin)
@@ -255,15 +269,16 @@ def ansible_command_sequence(configure_data_ansible, base_project, sequence, ver
             return False, f"Missing binary {ansible_bin}"
         ansible_bin_paths[ansible_bin] = binpath
 
+    # 3. Compose common parts of all ansible commands
     ansible_common = [ansible_bin_paths['ansible-playbook']]
     if verbose:
         ansible_common.append('-vvvv')
     else:
         ansible_common.append('-vv')
-
     ansible_common.append('-i')
     ansible_common.append(inventory)
 
+    # 4. Start composing and accumulating the list of all needed commands
     ansible_cmd = []
     ansible_cmd_seq = []
     ssh_share = ansible_common.copy()
@@ -275,23 +290,13 @@ def ansible_command_sequence(configure_data_ansible, base_project, sequence, ver
         'all', '-a', 'true',
         '--ssh-extra-args=-l cloudadmin -o UpdateHostKeys=yes -o StrictHostKeyChecking=accept-new'])
     ansible_cmd_seq.append({'cmd': ssh_share})
-    original_env = dict(os.environ)
-    original_env['ANSIBLE_PIPELINING'] = 'True'
-    ansible_callbacks = []
-    if profile:
-        ansible_callbacks.append('ansible.posix.profile_tasks')
-    if junit:
-        ansible_callbacks.append('junit')
-        original_env['JUNIT_OUTPUT_DIR'] = junit
-    if len(ansible_callbacks) > 0:
-        original_env['ANSIBLE_CALLBACKS_ENABLED'] = ','.join(ansible_callbacks)
-    if 'roles_path' in configure_data_ansible:
-        original_env['ANSIBLE_ROLES_PATH'] = configure_data_ansible['roles_path']
 
     for playbook in configure_data_ansible[sequence]:
         ansible_cmd = ansible_common.copy()
         playbook_cmd = playbook.split(' ')
         log.debug("playbook:%s", playbook)
+        # get the file named in the conf.yaml from playbook_cmd
+        # and append the full path within the repo folder
         playbook_filename = os.path.join(base_project, 'ansible', 'playbooks', playbook_cmd[0])
         ansible_cmd.append(playbook_filename)
         for ply_cmd in playbook_cmd[1:]:
@@ -310,15 +315,26 @@ def ansible_export_output(command, out):
     """ Write the Ansible (or ansible-playbook) stdout to file
 
     Function is in charge to:
-    - get a cmd and calculate from it the log file name to write. The filename is calculated, when available, from the playbook name: stripping '.yaml' and adding '.log.txt'
+    - get a cmd and calculate from it the log file name to write.
+      The filename is calculated, when available, from the playbook name: stripping '.yaml' and adding '.log.txt'
     - open a file in write mode. Path for this file is the current directory
     - write to the file the content of the out variable. Each element of the out list to a new file line
 
     Args:
         command (str list): one cmd element as prepared by ansible_command_sequence
         out (str list): as returned by subprocess_run
     """
-    playbook_path = command[4]
+    # log name has to be derived from the name of the playbook:
+    # search the playbook name in all command words.
+    playbook_path = None
+    for cmd_element in command:
+        match = re.search(rf"{os.path.join('ansible', 'playbooks')}.*", cmd_element)
+        if match:
+            playbook_path = cmd_element
+            break
+    if playbook_path is None:
+        log.error("Unable to find which one is the playbook in %s", command)
+        return
     playbook_name = os.path.splitext(os.path.basename(playbook_path))[0]
     log_filename = f"ansible.{playbook_name}.log.txt"
     log.debug("Write %s getcwd:%s", log_filename, os.getcwd())
@@ -374,7 +390,9 @@ def cmd_ansible(configure_data, base_project, dryrun, verbose, destroy=False, pr
         else:
             ret, out = lib.process_manager.subprocess_run(**command)
             log.debug("Ansible process return ret:%d", ret)
-            ansible_export_output(command['cmd'], out)
+            # only write separated files for ansible-playbook commands
+            if 'ansible-playbook' in command['cmd'][0]:
+                ansible_export_output(command['cmd'], out)
             if ret != 0:
                 log.error("command:%s returned non zero %d", command, ret)
                 return Status(f"Error rc: {ret} at {command}")

scripts/qesap/test/e2e/test.sh

@@ -309,28 +309,28 @@ grep -E "TASK.*Say hello" ansible.sambuconero.log.txt || test_die "Expected cont
 rm "${THIS_LOG}"
 rm ansible.*.log.txt
 
-test_step "[test_6.yaml] Check redirection to file of Ansible logs"
+test_step "[test_6.yaml] Check redirection to file of ansible-playbook logs"
 rm ansible.*.log.txt || echo "Nothing to delete"
 cp sambuconero.yaml "${QESAPROOT}/ansible/playbooks/timbio.yaml"
 cp sambuconero.yaml "${QESAPROOT}/ansible/playbooks/buga.yaml"
 cp sambuconero.yaml "${QESAPROOT}/ansible/playbooks/purace.yaml"
 qesap.py -b ${QESAPROOT} -c test_6.yaml ansible || test_die "test_6.yaml fail on ansible"
 ansible_logs_number=$(find . -type f -name "ansible.*.log.txt" | wc -l)
 echo "--> ansible_logs_number:${ansible_logs_number}"
-# 3 playbooks plus a log file for the initial ansible (not ansible-playbook) call 
-[[ $ansible_logs_number -eq 4 ]] || test_die "ansible .log.txt are not 4 files but has ${ansible_logs_number}"
+# 3 playbooks means 3 logs
+[[ $ansible_logs_number -eq 3 ]] || test_die "ansible .log.txt are not 3 files but has ${ansible_logs_number}"
 rm ansible.*.log.txt
 
-test_step "[test_7.yaml] Check redirection to file of Ansible logs in case of error in the playbook execution"
+test_step "[test_7.yaml] Check redirection to file of ansible-playbook logs in case of error in the playbook execution"
 rm ansible.*.log.txt || echo "Nothing to delete"
 cp marasca.yaml "${QESAPROOT}/ansible/playbooks/"
 set +e
 qesap.py --verbose -b ${QESAPROOT} -c test_7.yaml ansible
 rc=$?; [[ $rc -ne 0 ]] || test_die "qesap.py ansible has to fail if ansible-playbook fails rc:$rc"
 set -e
 ansible_logs_number=$(find . -type f -name "ansible.*.log.txt" | wc -l)
-# 2 playbooks plus a log file for the initial ansible (not ansible-playbook) call
-[[ $ansible_logs_number -eq 3 ]] || test_die "ansible .log.txt are not 3 files but has ${ansible_logs_number}"
+# 2 playbooks means 2 logs
+[[ $ansible_logs_number -eq 2 ]] || test_die "ansible .log.txt are not 2 files but has ${ansible_logs_number}"
 grep -E "TASK.*Say hello" ansible.sambuconero.log.txt || test_die "Expected content not found in ansible.sambuconero.log.txt"
 grep -E "TASK.*This fails" ansible.marasca.log.txt || test_die "Expected content not found in ansible.marasca.log.txt"
 rm ansible.*.log.txt

scripts/qesap/test/unit/test_qesap_ansible.py

@@ -690,7 +690,8 @@ def test_ansible_junit(
     mock_call_ansibleplaybook,
 ):
     """
-    Test that --junit result in Ansible called with two additional env variables
+    Test that using '--junit' on the glue script command line
+    results in two additional env variables added at each ansible command execution
 
         ANSIBLE_CALLBACKS_ENABLED=junit
         JUNIT_OUTPUT_DIR="/something/somewhere"

scripts/qesap/test/unit/test_qesap_lib_cmd_ansible.py

@@ -29,7 +29,7 @@ def test_export_ansible_output():
         "-vvvv",
         "-i",
         "/root/qe-sap-deployment/terraform/aws/inventory.yaml",
-        "/some/immaginary/path/testAll.yaml",
+        "/some/immaginary/path/ansible/playbooks/testAll.yaml",
         "-e",
         "something=somevalue",
     ]