EEH test for vnic interfaces

Adding test to trigger EEH for vnic interfaces from vios.
The test does the following:

- change the active logport to the original logport
- login to the vios and find the respective backing device
- trigger EEH from vios
- check for the failover on the OS side to conclude if the eeh was successful or not.

Signed-off-by: Vaishnavi Bhat <[email protected]>

io/net/virt-net/network_virtualization.py

@@ -35,6 +35,8 @@
 from avocado.utils.network.hosts import LocalHost
 from avocado.utils.ssh import Session
 from avocado.utils import wait
+from pexpect import pxssh
+import re
 
 IS_POWER_NV = 'PowerNV' in open('/proc/cpuinfo', 'r').read()
 IS_KVM_GUEST = 'qemu' in open('/proc/cpuinfo', 'r').read()
@@ -119,6 +121,9 @@ def setUp(self):
         self.mac_id = [mac.replace(':', '') for mac in self.mac_id]
         self.netmask = self.params.get('netmasks', default=None).split(' ')
         self.peer_ip = self.params.get('peer_ip', default=None).split(' ')
+        self.host_public_ip = self.params.get('host_public_ip', default=None)
+        self.host_user = self.params.get('user_name', default=None)
+        self.host_password = self.params.get('host_password', default=None)
         dmesg.clear_dmesg()
         self.session_hmc.cmd("uname -a")
         cmd = 'lssyscfg -m ' + self.server + \
@@ -594,6 +599,68 @@ def test_vnic_hmc_dlpar(self):
                 count += 1
         self.check_dmesg_error()
 
+    def test_vnic_eeh(self):
+        """
+        Perform EEH on vnic interface from vios
+        """
+        if self.backing_dev_count() == 1:
+            self.cancel("EEH cannot be tested as the interface has single backing device")
+        current_logport = self.get_active_device_logport(self.slot_num[0])
+        if not self.original_logport == current_logport:
+            self.trigger_failover(self.original_logport)
+        else:
+            self.log.info("Unable to set the logport to original one")
+        time.sleep(5)
+        self.session = Session(self.vios_ip, user=self.vios_user,
+                               password=self.vios_pwd)
+        self.session.cleanup_master()
+        if not wait.wait_for(self.session.connect, timeout=30):
+            self.fail("Failed connecting to VIOS")
+        cmd = "ioscli lsmap -all -vnic -cpid %s" % self.lpar_id
+        vnic_servers = self.session.cmd(cmd).stdout_text.splitlines()
+        device = self.find_device(self.mac_id[0])
+        temp_idx = vnic_servers.index("Client device name:" + device)
+        vnic_backingdevice = vnic_servers[temp_idx - 3].split(":")[1]
+        vios = pxssh.pxssh()
+        try:
+            vios.login(self.vios_ip, self.vios_user, self.vios_pwd)
+        except Exception:
+            self.warn("Unable to login to vios")
+        time.sleep(2)
+        vios.sendline("oem_setup_env")
+        time.sleep(5)
+        eeh_tool_64 = self.params.get('eeh_tool', default='eeh_tool_64')
+        eeh_tool_64 = self.get_data(eeh_tool_64)
+        cmd = "scp %s@%s:%s ." % (self.host_user, self.host_public_ip, eeh_tool_64)
+        vios.sendline(cmd)
+        time.sleep(3)
+        vios.sendline(self.host_password)
+        time.sleep(5)
+        vios.sendline("kdb")
+        time.sleep(5)
+        vios.sendline("set scroll false")
+        time.sleep(5)
+        cmd = "mlxcent setacs %s" % vnic_backingdevice
+        vios.sendline(cmd)
+        time.sleep(2)
+        vios.sendline("mlxcent pollq 0")
+        vios.prompt()
+        mapstart = vios.before.decode("utf-8").split("\r\n ")
+        for i in mapstart:
+            if re.search("map_start", i):
+                map_start_value = i.split("=")[1]
+        vios.sendline("quit")
+        cmd = "./eeh_tool_64 %s 3 15 -w 64 -a %s -m 0xFFFFFFFFFFFFF000" % (vnic_backingdevice, map_start_value)
+        vios.sendline(cmd)
+        time.sleep(5)
+        active_logport = self.get_active_device_logport(self.slot_num[0])
+        if current_logport == active_logport:
+            self.fail("EEH unsuccessful as there is no failover triggered on the OS")
+        device = self.find_device(self.mac_id[0])
+        networkinterface = NetworkInterface(device, self.local)
+        if networkinterface.ping_check(self.peer_ip[0], count=5) is not None:
+            self.fail("Ping to peer failed. EEH has affected Network connectivity")
+
     def backing_dev_count_w_slot_num(self, slot):
         """
         Lists the count of backing devices

io/net/virt-net/network_virtualization.py.data/README

@@ -16,6 +16,9 @@ RHEL :
 Links :
 -> https://pypi.python.org/packages/source/p/pycrypto/pycrypto-2.6.1.tar.gz#md5=55a61a054aa66812daf5161a0d5d7eda
 
+Install pexpect module 
+-> pip install pexpect 
+
 Explanation of the input parameters for single vnic:
 hmc_pwd ---> HMC password
 hmc_username ---> HMC user name
@@ -38,6 +41,9 @@ peer_ip ---> Peer ip to which the added Network virtualized interface will ping
 count ---> The number of times the failover test has to be executed
 num_of_dlpar --> number of times ddlpar remove and add operation to be executed
 mac_id ---> MAC ID to be set for the vnic interface. This is needed for us to have control over interface name via interface file or udev rules, space separated if multiple
+host_public_ip ---> Public IP of the host
+host_password ---> Login password for the host machine 
+user_name ---> Host user name 
 
 Explanation of the input parameters for multiple vnic:
 hmc_pwd ---> HMC password
@@ -55,6 +61,9 @@ device_ip ---> space separated ip address
 netmasks ---> space separated netmask
 peer_ip ---> space separated peer ip
 mac_id ---> MAC ID to be set for the vnic interface. This is needed for us to have control over interface name via interface file or udev rules, space separated mac id
+user_name ---> host user name 
+host_public_ip ---> public IP of the host, required to copy the eeh_tool to vios 
+host_password ---> Login password of the host, required to copy the eeh_tool to vios 
 
 NOTE: all failover test will not execute when it will create multiple Network virtualized interface.
 and for now multiple Network virtualized interface with multiple backing device will not work.

io/net/virt-net/network_virtualization.py.data/network_virtualization.yaml

@@ -19,3 +19,6 @@ peer_ip:
 count:
 num_of_dlpar:
 mac_id:
+user_name: ""
+host_public_ip: ""
+host_password: ""