EEH test for vnic interfaces

Adding test to trigger EEH for vnic interfaces from vios. The test does the following:

change the active logport to the original logport
login to the vios and find the respective backing device
trigger EEH from vios
check for the failover on the OS side to conclude if the eeh was successful or not.

Signed-off-by: Vaishnavi Bhat <[email protected]>

io/net/virt-net/network_virtualization.py

@@ -35,6 +35,9 @@
 from avocado.utils.network.hosts import LocalHost
 from avocado.utils.ssh import Session
 from avocado.utils import wait
+import pexpect
+from pexpect import pxssh
+import re
 
 IS_POWER_NV = 'PowerNV' in open('/proc/cpuinfo', 'r').read()
 IS_KVM_GUEST = 'qemu' in open('/proc/cpuinfo', 'r').read()
@@ -119,6 +122,9 @@ def setUp(self):
         self.mac_id = [mac.replace(':', '') for mac in self.mac_id]
         self.netmask = self.params.get('netmasks', default=None).split(' ')
         self.peer_ip = self.params.get('peer_ip', default=None).split(' ')
+        self.host_public_ip = self.params.get('host_public_ip', default=None)
+        self.host_user = self.params.get('user_name', default=None)
+        self.host_password = self.params.get('host_password', default=None)
         dmesg.clear_dmesg()
         self.session_hmc.cmd("uname -a")
         cmd = 'lssyscfg -m ' + self.server + \
@@ -594,6 +600,68 @@ def test_vnic_hmc_dlpar(self):
                 count += 1
         self.check_dmesg_error()
 
+    def test_vnic_eeh(self):
+        """
+        Perform EEH on vnic interface from vios
+        """
+        if self.backing_dev_count() == 1:
+            self.cancel("EEH cannot be tested as the interface has single backing device")
+        current_logport = self.get_active_device_logport(self.slot_num[0])
+        if not self.original_logport == current_logport:
+            self.trigger_failover(self.original_logport)
+        else:
+            self.log.info("Unable to set the logport to original one")
+        time.sleep(5)
+        self.session = Session(self.vios_ip, user=self.vios_user,
+                               password=self.vios_pwd)
+        self.session.cleanup_master()
+        if not wait.wait_for(self.session.connect, timeout=30):
+            self.fail("Failed connecting to VIOS")
+        cmd = "ioscli lsmap -all -vnic -cpid %s" % self.lpar_id
+        vnic_servers = self.session.cmd(cmd).stdout_text.splitlines()
+        device = self.find_device(self.mac_id[0])
+        temp_idx = vnic_servers.index("Client device name:" + device)
+        vnic_backingdevice = vnic_servers[temp_idx - 3].split(":")[1]
+        vios = pxssh.pxssh()
+        try:
+            vios.login(self.vios_ip, self.vios_user, self.vios_pwd)
+        except Exception:
+            self.warn("Unable to login to vios")
+        time.sleep(2)
+        vios.sendline("oem_setup_env")
+        time.sleep(5)
+        eeh_tool_64 = self.params.get('eeh_tool', default='eeh_tool_64')
+        eeh_tool_64 = self.get_data(eeh_tool_64)
+        cmd = "scp %s@%s:%s ." % (self.host_user, self.host_public_ip, eeh_tool_64)
+        vios.sendline(cmd)
+        time.sleep(3)
+        vios.sendline(self.host_password)
+        time.sleep(5)
+        vios.sendline("kdb")
+        time.sleep(5)
+        vios.sendline("set scroll false")
+        time.sleep(5)
+        cmd = "mlxcent setacs %s" % vnic_backingdevice
+        vios.sendline(cmd)
+        time.sleep(2)
+        vios.sendline("mlxcent pollq 0")
+        vios.prompt()
+        mapstart = vios.before.decode("utf-8").split("\r\n ")
+        for i in mapstart:
+            if re.search("map_start", i):
+                map_start_value = i.split("=")[1]
+        vios.sendline("quit")
+        cmd = "./eeh_tool_64 %s 3 15 -w 64 -a %s -m 0xFFFFFFFFFFFFF000" % (vnic_backingdevice, map_start_value)
+        vios.sendline(cmd)
+        time.sleep(5)
+        active_logport = self.get_active_device_logport(self.slot_num[0])
+        if current_logport == active_logport:
+            self.fail("EEH unsuccessful as there is no failover triggered on the OS")
+        device = self.find_device(self.mac_id[0])
+        networkinterface = NetworkInterface(device, self.local)
+        if networkinterface.ping_check(self.peer_ip[0], count=5) is not None:
+            self.fail("Ping to peer failed. EEH has affected Network connectivity")
+
     def backing_dev_count_w_slot_num(self, slot):
         """
         Lists the count of backing devices

io/net/virt-net/network_virtualization.py.data/README

@@ -16,6 +16,9 @@ RHEL :
 Links :
 -> https://pypi.python.org/packages/source/p/pycrypto/pycrypto-2.6.1.tar.gz#md5=55a61a054aa66812daf5161a0d5d7eda
 
+Install pexpect module 
+-> pip install pexpect 
+
 Explanation of the input parameters for single vnic:
 hmc_pwd ---> HMC password
 hmc_username ---> HMC user name
@@ -38,6 +41,9 @@ peer_ip ---> Peer ip to which the added Network virtualized interface will ping
 count ---> The number of times the failover test has to be executed
 num_of_dlpar --> number of times ddlpar remove and add operation to be executed
 mac_id ---> MAC ID to be set for the vnic interface. This is needed for us to have control over interface name via interface file or udev rules, space separated if multiple
+host_public_ip ---> Public IP of the host
+host_password ---> Login password for the host machine 
+user_name ---> Host user name 
 
 Explanation of the input parameters for multiple vnic:
 hmc_pwd ---> HMC password
@@ -55,6 +61,9 @@ device_ip ---> space separated ip address
 netmasks ---> space separated netmask
 peer_ip ---> space separated peer ip
 mac_id ---> MAC ID to be set for the vnic interface. This is needed for us to have control over interface name via interface file or udev rules, space separated mac id
+user_name ---> host user name 
+host_public_ip ---> public IP of the host, required to copy the eeh_tool to vios 
+host_password ---> Login password of the host, required to copy the eeh_tool to vios 
 
 NOTE: all failover test will not execute when it will create multiple Network virtualized interface.
 and for now multiple Network virtualized interface with multiple backing device will not work.

io/net/virt-net/network_virtualization.py.data/network_virtualization.yaml

@@ -19,3 +19,6 @@ peer_ip:
 count:
 num_of_dlpar:
 mac_id:
+user_name: ""
+host_public_ip: ""
+host_password: ""