From 5170a816ec3ff9fa28e81986a17f45784553e068 Mon Sep 17 00:00:00 2001 From: Balamuruhan S Date: Tue, 20 Aug 2019 13:09:14 +0530 Subject: [PATCH] utils_misc: Reorg disk related APIs into utils_disk Reorganise disk related method to utils_disk and break circular imports caused with respect to disk related APIs in utils_misc and gluster to utils_disk for better work flow. Rename utils_disk.py to disk.py Signed-off-by: Srikanth Aithal Signed-off-by: Balamuruhan S --- selftests/unit/test_nfs.py | 20 +- virttest/{utils_disk.py => disk.py} | 343 ++++++++++++++++++++++++++- virttest/gluster.py | 113 --------- virttest/nfs.py | 9 +- virttest/storage.py | 3 +- virttest/tests/unattended_install.py | 7 +- virttest/utils_misc.py | 286 +--------------------- virttest/utils_v2v.py | 19 +- 8 files changed, 380 insertions(+), 420 deletions(-) rename virttest/{utils_disk.py => disk.py} (79%) diff --git a/selftests/unit/test_nfs.py b/selftests/unit/test_nfs.py index 77d0720d6ad..ea98e50d169 100644 --- a/selftests/unit/test_nfs.py +++ b/selftests/unit/test_nfs.py @@ -14,7 +14,7 @@ from virttest.unittest_utils import mock from virttest import nfs -from virttest import utils_misc +from virttest import utils_disk from virttest.staging import service @@ -58,16 +58,16 @@ def setup_stubs_setup(self, nfs_obj): os.makedirs.expect_call(nfs_obj.export_dir) nfs_obj.exportfs.export.expect_call() os.makedirs.expect_call(nfs_obj.mount_dir) - utils_misc.mount.expect_call(nfs_obj.mount_src, nfs_obj.mount_dir, - "nfs", perm=nfs_obj.mount_options) + utils_disk.mount.expect_call(nfs_obj.mount_src, nfs_obj.mount_dir, + "nfs", nfs_obj.mount_options) def setup_stubs_is_mounted(self, nfs_obj): - utils_misc.is_mounted.expect_call(nfs_obj.mount_src, - nfs_obj.mount_dir, - "nfs").and_return(True) + utils_disk.is_mount.expect_call(nfs_obj.mount_src, + nfs_obj.mount_dir, + "nfs").and_return(True) def setup_stubs_cleanup(self, nfs_obj): - utils_misc.umount.expect_call(nfs_obj.mount_src, + utils_disk.umount.expect_call(nfs_obj.mount_src, nfs_obj.mount_dir, "nfs") nfs_obj.exportfs.reset_export.expect_call() @@ -84,9 +84,9 @@ def setUp(self): self.god.stub_function(process, "system_output") self.god.stub_function(os.path, "isfile") self.god.stub_function(os, "makedirs") - self.god.stub_function(utils_misc, "is_mounted") - self.god.stub_function(utils_misc, "mount") - self.god.stub_function(utils_misc, "umount") + self.god.stub_function(utils_disk, "is_mount") + self.god.stub_function(utils_disk, "mount") + self.god.stub_function(utils_disk, "umount") self.god.stub_function(service.Factory, "create_service") attr = getattr(nfs, "Exportfs") setattr(attr, "already_exported", False) diff --git a/virttest/utils_disk.py b/virttest/disk.py similarity index 79% rename from virttest/utils_disk.py rename to virttest/disk.py index 04155c98172..4ea569881b4 100644 --- a/virttest/utils_disk.py +++ b/virttest/disk.py @@ -12,6 +12,7 @@ import tempfile import logging import re +import socket try: import configparser as ConfigParser except ImportError: @@ -20,10 +21,15 @@ from avocado.core import exceptions from avocado.utils import process +from avocado.utils import data_factory from avocado.utils.service import SpecificServiceManager from virttest import error_context from virttest import utils_numeric +from virttest import utils_net +from virttest import gluster +from virttest import data_dir +from virttest.libvirt_xml import pool_xml from virttest.compat_52lts import decode_to_text from virttest.compat_52lts import results_stdout_52lts @@ -67,7 +73,7 @@ def copytree(src, dst, overwrite=True, ignore=''): def is_mount(src, dst=None, fstype=None, options=None, verbose=False, - session=None): + session=None, fstype_mtab=None): """ Check is src or dst mounted. @@ -109,7 +115,8 @@ def is_mount(src, dst=None, fstype=None, options=None, verbose=False, return False -def mount(src, dst, fstype=None, options=None, verbose=False, session=None): +def mount(src, dst, fstype=None, options=None, verbose=False, session=None, + fstype_mtab=None): """ Mount src under dst if it's really mounted, then remout with options. @@ -137,7 +144,8 @@ def mount(src, dst, fstype=None, options=None, verbose=False, session=None): return process.system(cmd, verbose=verbose) == 0 -def umount(src, dst, fstype=None, verbose=False, session=None): +def umount(src, dst, fstype=None, verbose=False, session=None, + fstype_mtab=None): """ Umount src from dst, if src really mounted under dst. @@ -1026,6 +1034,335 @@ def get_disk_by_serial(serial_str, session=None): return disk +def new_disk_vol_name(pool_name): + """ + According to BZ#1138523, the new volume name must be the next + created partition(sdb1, etc.), so we need to inspect the original + partitions of the disk then count the new partition number. + + :param pool_name: Disk pool name + :return: New volume name or none + """ + poolxml = pool_xml.PoolXML.new_from_dumpxml(pool_name) + if poolxml.get_type(pool_name) != "disk": + logging.error("This is not a disk pool") + return None + disk = poolxml.get_source().device_path[5:] + part_num = len(list(filter(lambda s: s.startswith(disk), + get_parts_list()))) + return disk + str(part_num) + + +def list_linux_guest_disks(session, partition=False): + """ + List all disks OR disks with no partition in linux guest. + + :param session: session object to guest + :param partition: if true, list all disks; otherwise, + list only disks with no partition. + :return: the disks set. + """ + cmd = "ls /dev/[vhs]d*" + if not partition: + cmd = "%s | grep -v [0-9]$" % cmd + status, output = session.cmd_status_output(cmd) + if status != 0: + raise exceptions.TestFail("Get disks failed with output %s" % output) + return set(output.split()) + + +def get_all_disks_did(session, partition=False): + """ + Get all disks did lists in a linux guest, each disk list + include disk kname, serial and wwn. + + :param session: session object to guest. + :param partition: if true, get all disks did lists; otherwise, + get the ones with no partition. + :return: a dict with all disks did lists each include disk + kname, serial and wwn. + """ + disks = list_linux_guest_disks(session, partition) + logging.debug("Disks detail: %s" % disks) + all_disks_did = {} + for line in disks: + kname = line.split('/')[2] + get_disk_info_cmd = "udevadm info -q property -p /sys/block/%s" % kname + output = session.cmd_output_safe(get_disk_info_cmd) + re_str = r"(?<=DEVNAME=/dev/)(.*)|(?<=ID_SERIAL=)(.*)|" + re_str += "(?<=ID_SERIAL_SHORT=)(.*)|(?<=ID_WWN=)(.*)" + did_list_group = re.finditer(re_str, output, re.M) + did_list = [match.group() for match in did_list_group if match] + all_disks_did[kname] = did_list + + return all_disks_did + + +def format_windows_disk(session, did, mountpoint=None, size=None, fstype="ntfs", + labletype=PARTITION_TABLE_TYPE_MBR, force=False): + """ + Create a partition on disk in windows guest and format it. + + :param session: session object to guest. + :param did: disk index which show in 'diskpart list disk'. + :param mountpoint: mount point for the disk. + :param size: partition size. + :param fstype: filesystem type for the disk. + :param labletype: disk partition table type. + :param force: if need force format. + :return Boolean: disk usable or not. + """ + list_disk_cmd = "echo list disk > disk && " + list_disk_cmd += "echo exit >> disk && diskpart /s disk" + disks = session.cmd_output(list_disk_cmd, timeout=120) + create_partition_table_windows(session, did, labletype) + + if size: + size = int(float(utils_numeric.normalize_data_size(size, + order_magnitude="M"))) + + for disk in disks.splitlines(): + if re.search(r"DISK %s" % did, disk, re.I | re.M): + cmd_header = 'echo list disk > disk &&' + cmd_header += 'echo select disk %s >> disk &&' % did + cmd_footer = '&& echo exit>> disk && diskpart /s disk' + cmd_footer += '&& del /f disk' + detail_cmd = 'echo detail disk >> disk' + detail_cmd = ' '.join([cmd_header, detail_cmd, cmd_footer]) + logging.debug("Detail for 'Disk%s'" % did) + details = session.cmd_output(detail_cmd) + + pattern = "DISK %s.*Offline" % did + if re.search(pattern, details, re.I | re.M): + online_cmd = 'echo online disk>> disk' + online_cmd = ' '.join([cmd_header, online_cmd, cmd_footer]) + logging.info("Online 'Disk%s'" % did) + session.cmd(online_cmd) + + if re.search("Read.*Yes", details, re.I | re.M): + set_rw_cmd = 'echo attributes disk clear readonly>> disk' + set_rw_cmd = ' '.join([cmd_header, set_rw_cmd, cmd_footer]) + logging.info("Clear readonly bit on 'Disk%s'" % did) + session.cmd(set_rw_cmd) + + if re.search(r"Volume.*%s" % fstype, details, re.I | re.M) and not force: + logging.info("Disk%s has been formated, cancel format" % did) + continue + + if not size: + mkpart_cmd = 'echo create partition primary >> disk' + else: + mkpart_cmd = 'echo create partition primary size=%s ' + mkpart_cmd += '>> disk' + mkpart_cmd = mkpart_cmd % size + list_cmd = ' && echo list partition >> disk ' + cmds = ' '.join([cmd_header, mkpart_cmd, list_cmd, cmd_footer]) + logging.info("Create partition on 'Disk%s'" % did) + partition_index = re.search( + r'\*\s+Partition\s(\d+)\s+', session.cmd(cmds), re.M).group(1) + logging.info("Format the 'Disk%s' to %s" % (did, fstype)) + format_cmd = 'echo list partition >> disk && ' + format_cmd += 'echo select partition %s >> disk && ' % partition_index + if not mountpoint: + format_cmd += 'echo assign >> disk && ' + else: + format_cmd += 'echo assign letter=%s >> disk && ' % mountpoint + format_cmd += 'echo format fs=%s quick >> disk ' % fstype + format_cmd = ' '.join([cmd_header, format_cmd, cmd_footer]) + session.cmd(format_cmd, timeout=300) + + return True + + return False + + +def format_linux_disk(session, did, all_disks_did, partition=False, + mountpoint=None, size=None, fstype="ext3"): + """ + Create a partition on disk in linux guest and format and mount it. + + :param session: session object to guest. + :param did: disk kname, serial or wwn. + :param all_disks_did: all disks did lists each include + disk kname, serial and wwn. + :param partition: if true, can format all disks; otherwise, + only format the ones with no partition originally. + :param mountpoint: mount point for the disk. + :param size: partition size. + :param fstype: filesystem type for the disk. + :return Boolean: disk usable or not. + """ + disks = list_linux_guest_disks(session, partition) + logging.debug("Disks detail: %s" % disks) + for line in disks: + kname = line.split('/')[2] + did_list = all_disks_did[kname] + if did not in did_list: + # Continue to search target disk + continue + if not size: + size_output = session.cmd_output_safe("lsblk -oKNAME,SIZE|grep %s" + % kname) + size = size_output.splitlines()[0].split()[1] + all_disks_before = list_linux_guest_disks(session, True) + devname = line + logging.info("Create partition on disk '%s'" % devname) + mkpart_cmd = "parted -s %s mklabel gpt mkpart " + mkpart_cmd += "primary 0 %s" + mkpart_cmd = mkpart_cmd % (devname, size) + session.cmd_output_safe(mkpart_cmd) + session.cmd_output_safe("partprobe %s" % devname) + all_disks_after = list_linux_guest_disks(session, True) + partname = (all_disks_after - all_disks_before).pop() + logging.info("Format partition to '%s'" % fstype) + format_cmd = "yes|mkfs -t %s %s" % (fstype, partname) + session.cmd_output_safe(format_cmd) + if not mountpoint: + session.cmd_output_safe("mkdir /mnt/%s" % kname) + mountpoint = os.path.join("/mnt", kname) + logging.info("Mount the disk to '%s'" % mountpoint) + mount_cmd = "mount -t %s %s %s" % (fstype, partname, mountpoint) + session.cmd_output_safe(mount_cmd) + return True + + return False + + +def format_guest_disk(session, did, all_disks_did, ostype, partition=False, + mountpoint=None, size=None, fstype=None): + """ + Create a partition on disk in guest and format and mount it. + + :param session: session object to guest. + :param did: disk ID in guest. + :param all_disks_did: a dict contains all disks did lists each + include disk kname, serial and wwn for linux guest. + :param ostype: guest os type 'windows' or 'linux'. + :param partition: if true, can format all disks; otherwise, + only format the ones with no partition originally. + :param mountpoint: mount point for the disk. + :param size: partition size. + :param fstype: filesystem type for the disk; when it's the default None, + it will use the default one for corresponding ostype guest + :return Boolean: disk usable or not. + """ + default_fstype = "ntfs" if (ostype == "windows") else "ext3" + fstype = fstype or default_fstype + if ostype == "windows": + return format_windows_disk(session, did, mountpoint, size, fstype) + return format_linux_disk(session, did, all_disks_did, partition, + mountpoint, size, fstype) + + +def glusterfs_mount(g_uri, mount_point): + """ + Mount gluster volume to mountpoint. + + :param g_uri: stripped gluster uri from create_gluster_uri(.., True) + :type g_uri: str + """ + mount(g_uri, mount_point, "glusterfs", None, False, "fuse.glusterfs") + + +@error_context.context_aware +def create_gluster_vol(params): + vol_name = params.get("gluster_volume_name") + force = params.get('force_recreate_gluster') == "yes" + + brick_path = params.get("gluster_brick") + if not os.path.isabs(brick_path): # do nothing when path is absolute + base_dir = params.get("images_base_dir", data_dir.get_data_dir()) + brick_path = os.path.join(base_dir, brick_path) + + error_context.context("Host name lookup failed") + hostname = socket.gethostname() + if not hostname or hostname == "(none)": + if_up = utils_net.get_net_if(state="UP") + for i in if_up: + ipv4_value = utils_net.get_net_if_addrs(i)["ipv4"] + logging.debug("ipv4_value is %s", ipv4_value) + if ipv4_value != []: + ip_addr = ipv4_value[0] + break + hostname = ip_addr + + # Start the gluster dameon, if not started + gluster.glusterd_start() + # Check for the volume is already present, if not create one. + if not gluster.is_gluster_vol_avail(vol_name) or force: + return gluster.gluster_vol_create(vol_name, hostname, brick_path, + force) + else: + return True + + +@error_context.context_aware +def gluster_vol_create(vol_name, hostname, brick_path, force=False, session=None): + """ + Gluster Volume Creation + + :param vol_name: Name of gluster volume + :param hostname: hostname to create gluster volume + :param force: Boolean for adding force option or not + """ + # Create a brick + if gluster.is_gluster_vol_avail(vol_name, session): + gluster.gluster_vol_stop(vol_name, True, session) + gluster.gluster_vol_delete(vol_name, session) + gluster.gluster_brick_delete(brick_path, session) + + gluster.gluster_brick_create(brick_path, session=session) + + if force: + force_opt = "force" + else: + force_opt = "" + + cmd = "gluster volume create %s %s:/%s %s" % (vol_name, hostname, + brick_path, force_opt) + error_context.context("Volume creation failed") + if session: + session.cmd(cmd) + else: + process.system(cmd) + return gluster.is_gluster_vol_avail(vol_name, session) + + +def file_exists(params, filename_path): + sg_uri = gluster.create_gluster_uri(params, stripped=True) + g_uri = gluster.create_gluster_uri(params, stripped=False) + # Using directly /tmp dir because directory should be really temporary and + # should be deleted immediately when no longer needed and + # created directory don't file tmp dir by any data. + tmpdir = "gmount-%s" % (data_factory.generate_random_string(6)) + tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir) + while os.path.exists(tmpdir_path): + tmpdir = "gmount-%s" % (data_factory.generate_random_string(6)) + tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir) + ret = False + try: + try: + os.mkdir(tmpdir_path) + glusterfs_mount(sg_uri, tmpdir_path) + mount_filename_path = os.path.join(tmpdir_path, + filename_path[len(g_uri):]) + if os.path.exists(mount_filename_path): + ret = True + except Exception as e: + logging.error("Failed to mount gluster volume %s to" + " mount dir %s: %s", sg_uri, tmpdir_path, e) + finally: + if umount(sg_uri, tmpdir_path, "glusterfs", False, "fuse.glusterfs"): + try: + os.rmdir(tmpdir_path) + except OSError: + pass + else: + logging.warning("Unable to unmount tmp directory %s with glusterfs" + " mount.", tmpdir_path) + return ret + + class Disk(object): """ diff --git a/virttest/gluster.py b/virttest/gluster.py index ceef1138dff..ea7e11bba9f 100644 --- a/virttest/gluster.py +++ b/virttest/gluster.py @@ -7,14 +7,11 @@ """ import logging -import os import re import socket from avocado.utils import process -from virttest import data_dir -from virttest import utils_misc from virttest import utils_net from virttest import error_context from virttest.compat_52lts import decode_to_text @@ -204,80 +201,6 @@ def gluster_brick_delete(brick_path, session=None): logging.error("Not able to delete brick folder %s", details) -@error_context.context_aware -def gluster_vol_create(vol_name, hostname, brick_path, force=False, session=None): - """ - Gluster Volume Creation - - :param vol_name: Name of gluster volume - :param hostname: hostname to create gluster volume - :param force: Boolean for adding force option or not - """ - # Create a brick - if is_gluster_vol_avail(vol_name, session): - gluster_vol_stop(vol_name, True, session) - gluster_vol_delete(vol_name, session) - gluster_brick_delete(brick_path, session) - - gluster_brick_create(brick_path, session=session) - - if force: - force_opt = "force" - else: - force_opt = "" - - cmd = "gluster volume create %s %s:/%s %s" % (vol_name, hostname, - brick_path, force_opt) - error_context.context("Volume creation failed") - if session: - session.cmd(cmd) - else: - process.system(cmd) - return is_gluster_vol_avail(vol_name, session) - - -def glusterfs_mount(g_uri, mount_point): - """ - Mount gluster volume to mountpoint. - - :param g_uri: stripped gluster uri from create_gluster_uri(.., True) - :type g_uri: str - """ - utils_misc.mount(g_uri, mount_point, "glusterfs", None, - False, "fuse.glusterfs") - - -@error_context.context_aware -def create_gluster_vol(params): - vol_name = params.get("gluster_volume_name") - force = params.get('force_recreate_gluster') == "yes" - - brick_path = params.get("gluster_brick") - if not os.path.isabs(brick_path): # do nothing when path is absolute - base_dir = params.get("images_base_dir", data_dir.get_data_dir()) - brick_path = os.path.join(base_dir, brick_path) - - error_context.context("Host name lookup failed") - hostname = socket.gethostname() - if not hostname or hostname == "(none)": - if_up = utils_net.get_net_if(state="UP") - for i in if_up: - ipv4_value = utils_net.get_net_if_addrs(i)["ipv4"] - logging.debug("ipv4_value is %s", ipv4_value) - if ipv4_value != []: - ip_addr = ipv4_value[0] - break - hostname = ip_addr - - # Start the gluster dameon, if not started - glusterd_start() - # Check for the volume is already present, if not create one. - if not is_gluster_vol_avail(vol_name) or force: - return gluster_vol_create(vol_name, hostname, brick_path, force) - else: - return True - - @error_context.context_aware def create_gluster_uri(params, stripped=False): """ @@ -307,42 +230,6 @@ def create_gluster_uri(params, stripped=False): return gluster_uri -def file_exists(params, filename_path): - sg_uri = create_gluster_uri(params, stripped=True) - g_uri = create_gluster_uri(params, stripped=False) - # Using directly /tmp dir because directory should be really temporary and - # should be deleted immediately when no longer needed and - # created directory don't file tmp dir by any data. - tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6)) - tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir) - while os.path.exists(tmpdir_path): - tmpdir = "gmount-%s" % (utils_misc.generate_random_string(6)) - tmpdir_path = os.path.join(data_dir.get_tmp_dir(), tmpdir) - ret = False - try: - try: - os.mkdir(tmpdir_path) - glusterfs_mount(sg_uri, tmpdir_path) - mount_filename_path = os.path.join(tmpdir_path, - filename_path[len(g_uri):]) - if os.path.exists(mount_filename_path): - ret = True - except Exception as e: - logging.error("Failed to mount gluster volume %s to" - " mount dir %s: %s" % (sg_uri, tmpdir_path, e)) - finally: - if utils_misc.umount(sg_uri, tmpdir_path, "glusterfs", False, - "fuse.glusterfs"): - try: - os.rmdir(tmpdir_path) - except OSError: - pass - else: - logging.warning("Unable to unmount tmp directory %s with glusterfs" - " mount.", tmpdir_path) - return ret - - def get_image_filename(params, image_name, image_format): """ Form the image file name using gluster uri diff --git a/virttest/nfs.py b/virttest/nfs.py index 7bdc70b8f6f..43b524639d0 100644 --- a/virttest/nfs.py +++ b/virttest/nfs.py @@ -13,6 +13,7 @@ from virttest import utils_misc from virttest import test_setup +from virttest import utils_disk from virttest.utils_iptables import Iptables from virttest.utils_conn import SSHConnection from virttest.compat_52lts import results_stdout_52lts, results_stderr_52lts @@ -212,20 +213,20 @@ def is_mounted(self): :return: If the src is mounted as expect :rtype: Boolean """ - return utils_misc.is_mounted(self.mount_src, self.mount_dir, "nfs") + return utils_disk.is_mount(self.mount_src, self.mount_dir, "nfs") def mount(self): """ Mount source into given mount point. """ - return utils_misc.mount(self.mount_src, self.mount_dir, "nfs", - perm=self.mount_options) + return utils_disk.mount(self.mount_src, self.mount_dir, "nfs", + self.mount_options) def umount(self): """ Umount the given mount point. """ - return utils_misc.umount(self.mount_src, self.mount_dir, "nfs") + return utils_disk.umount(self.mount_src, self.mount_dir, "nfs") def setup(self): """ diff --git a/virttest/storage.py b/virttest/storage.py index 3f5e17384d2..3dd88e54fb6 100644 --- a/virttest/storage.py +++ b/virttest/storage.py @@ -14,6 +14,7 @@ from avocado.core import exceptions from avocado.utils import process +from avocado.utils import data_factory from virttest import iscsi from virttest import utils_misc @@ -395,7 +396,7 @@ def get_backup_set(filename, backup_dir, action, good): # of the good image src_bad = filename src_good = os.path.join(backup_dir, "%s.backup" % basename) - hsh = utils_misc.generate_random_string(4) + hsh = data_factory.generate_random_string(4) dst_bad = (os.path.join(backup_dir, "%s.bad.%s" % (basename, hsh))) dst_good = (os.path.join(backup_dir, "%s.good.%s" % diff --git a/virttest/tests/unattended_install.py b/virttest/tests/unattended_install.py index a22f9cd0e21..f366e5b9b98 100644 --- a/virttest/tests/unattended_install.py +++ b/virttest/tests/unattended_install.py @@ -18,6 +18,7 @@ from avocado.utils import process from avocado.utils import crypto from avocado.utils import download +from avocado.utils import wait from virttest import virt_vm from virttest import asset @@ -1129,10 +1130,10 @@ def terminate_syslog_server_thread(): def copy_file_from_nfs(src, dst, mount_point, image_name): logging.info("Test failed before the install process start." " So just copy a good image from nfs for following tests.") - utils_misc.mount(src, mount_point, "nfs", perm="ro") + utils_disk.mount(src, mount_point, "nfs", "ro") image_src = utils_misc.get_path(mount_point, image_name) shutil.copy(image_src, dst) - utils_misc.umount(src, mount_point, "nfs") + utils_disk.umount(src, mount_point, "nfs") def string_in_serial_log(serial_log_file_path, string): @@ -1478,7 +1479,7 @@ def copy_images(): if params.get("medium", "cdrom") == "import": vm.shutdown() try: - if utils_misc.wait_for(vm.is_dead, shutdown_cleanly_timeout, 1, 1): + if wait.wait_for(vm.is_dead, shutdown_cleanly_timeout, 1, 1): logging.info("Guest managed to shutdown cleanly") except qemu_monitor.MonitorError as e: logging.warning("Guest apparently shut down, but got a " diff --git a/virttest/utils_misc.py b/virttest/utils_misc.py index fc25cc43a6c..ed4c08e142c 100644 --- a/virttest/utils_misc.py +++ b/virttest/utils_misc.py @@ -7,7 +7,6 @@ from __future__ import division import time import string -import random import socket import os import stat @@ -54,6 +53,7 @@ from avocado.utils import download from avocado.utils import linux_modules from avocado.utils import memory +from avocado.utils import data_factory from avocado.utils.astring import string_safe_encode # Symlink avocado implementation of process functions from avocado.utils.process import CmdResult @@ -71,7 +71,6 @@ from virttest import error_context from virttest import cartesian_config from virttest import utils_selinux -from virttest import utils_disk from virttest import logging_manager from virttest import libvirt_version from virttest.staging import utils_koji @@ -495,39 +494,11 @@ def get_path(base_path, user_path): return os.path.realpath(user_path) -def generate_random_string(length, ignore_str=string.punctuation, - convert_str=""): - """ - Return a random string using alphanumeric characters. - - :param length: Length of the string that will be generated. - :param ignore_str: Characters that will not include in generated string. - :param convert_str: Characters that need to be escaped (prepend "\\"). - - :return: The generated random string. - """ - r = random.SystemRandom() - sr = "" - chars = string.ascii_letters + string.digits + string.punctuation - if not ignore_str: - ignore_str = "" - for i in ignore_str: - chars = chars.replace(i, "") - - while length > 0: - tmp = r.choice(chars) - if convert_str and (tmp in convert_str): - tmp = "\\%s" % tmp - sr += tmp - length -= 1 - return sr - - def generate_random_id(): """ Return a random string suitable for use as a qemu id. """ - return "id" + generate_random_string(6) + return "id" + data_factory.generate_random_string(6) def generate_tmp_file_name(file_name, ext=None, @@ -537,7 +508,7 @@ def generate_tmp_file_name(file_name, ext=None, """ while True: file_name = (file_name + '-' + time.strftime("%Y%m%d-%H%M%S-") + - generate_random_string(4)) + data_factory.generate_random_string(4)) if ext: file_name += '.' + ext file_name = os.path.join(directory, file_name) @@ -1027,57 +998,6 @@ def safe_rmdir(path, timeout=10, session=None): (path, timeout, attempts)) -def umount(src, mount_point, fstype, verbose=False, fstype_mtab=None): - """ - Umount the src mounted in mount_point. - - :src: mount source - :mount_point: mount point - :type: file system type - :param fstype_mtab: file system type in mtab could be different - :type fstype_mtab: str - """ - return utils_disk.umount(src, mount_point, fstype, verbose) - - -def mount(src, mount_point, fstype, perm=None, verbose=False, fstype_mtab=None): - """ - Mount the src into mount_point of the host. - - :src: mount source - :mount_point: mount point - :fstype: file system type - :perm: mount permission - :param fstype_mtab: file system type in mtab could be different - :type fstype_mtab: str - """ - return utils_disk.mount(src, mount_point, fstype, perm, verbose) - - -def is_mounted(src, mount_point, fstype, perm=None, verbose=False, - fstype_mtab=None, session=None): - """ - Check mount status from /etc/mtab - - :param src: mount source - :type src: string - :param mount_point: mount point - :type mount_point: string - :param fstype: file system type - :type fstype: string - :param perm: mount permission - :type perm: string - :param verbose: if display mtab content - :type verbose: Boolean - :param fstype_mtab: file system type in mtab could be different - :type fstype_mtab: str - :param session: Session Object - :return: if the src is mounted as expect - :rtype: Boolean - """ - return utils_disk.is_mount(src, mount_point, fstype, perm, verbose, session) - - def install_host_kernel(job, params): """ Install a host kernel, given the appropriate params. @@ -2837,206 +2757,6 @@ def get_uptime(session=None): return float(uptime.split()[0]) -def list_linux_guest_disks(session, partition=False): - """ - List all disks OR disks with no partition in linux guest. - - :param session: session object to guest - :param partition: if true, list all disks; otherwise, - list only disks with no partition. - :return: the disks set. - """ - cmd = "ls /dev/[vhs]d*" - if not partition: - cmd = "%s | grep -v [0-9]$" % cmd - status, output = session.cmd_status_output(cmd) - if status != 0: - raise exceptions.TestFail("Get disks failed with output %s" % output) - return set(output.split()) - - -def get_all_disks_did(session, partition=False): - """ - Get all disks did lists in a linux guest, each disk list - include disk kname, serial and wwn. - - :param session: session object to guest. - :param partition: if true, get all disks did lists; otherwise, - get the ones with no partition. - :return: a dict with all disks did lists each include disk - kname, serial and wwn. - """ - disks = list_linux_guest_disks(session, partition) - logging.debug("Disks detail: %s" % disks) - all_disks_did = {} - for line in disks: - kname = line.split('/')[2] - get_disk_info_cmd = "udevadm info -q property -p /sys/block/%s" % kname - output = session.cmd_output_safe(get_disk_info_cmd) - re_str = r"(?<=DEVNAME=/dev/)(.*)|(?<=ID_SERIAL=)(.*)|" - re_str += "(?<=ID_SERIAL_SHORT=)(.*)|(?<=ID_WWN=)(.*)" - did_list_group = re.finditer(re_str, output, re.M) - did_list = [match.group() for match in did_list_group if match] - all_disks_did[kname] = did_list - - return all_disks_did - - -def format_windows_disk(session, did, mountpoint=None, size=None, fstype="ntfs", - labletype=utils_disk.PARTITION_TABLE_TYPE_MBR, force=False): - """ - Create a partition on disk in windows guest and format it. - - :param session: session object to guest. - :param did: disk index which show in 'diskpart list disk'. - :param mountpoint: mount point for the disk. - :param size: partition size. - :param fstype: filesystem type for the disk. - :param labletype: disk partition table type. - :param force: if need force format. - :return Boolean: disk usable or not. - """ - list_disk_cmd = "echo list disk > disk && " - list_disk_cmd += "echo exit >> disk && diskpart /s disk" - disks = session.cmd_output(list_disk_cmd, timeout=120) - utils_disk.create_partition_table_windows(session, did, labletype) - - if size: - size = int(float(normalize_data_size(size, order_magnitude="M"))) - - for disk in disks.splitlines(): - if re.search(r"DISK %s" % did, disk, re.I | re.M): - cmd_header = 'echo list disk > disk &&' - cmd_header += 'echo select disk %s >> disk &&' % did - cmd_footer = '&& echo exit>> disk && diskpart /s disk' - cmd_footer += '&& del /f disk' - detail_cmd = 'echo detail disk >> disk' - detail_cmd = ' '.join([cmd_header, detail_cmd, cmd_footer]) - logging.debug("Detail for 'Disk%s'" % did) - details = session.cmd_output(detail_cmd) - - pattern = "DISK %s.*Offline" % did - if re.search(pattern, details, re.I | re.M): - online_cmd = 'echo online disk>> disk' - online_cmd = ' '.join([cmd_header, online_cmd, cmd_footer]) - logging.info("Online 'Disk%s'" % did) - session.cmd(online_cmd) - - if re.search("Read.*Yes", details, re.I | re.M): - set_rw_cmd = 'echo attributes disk clear readonly>> disk' - set_rw_cmd = ' '.join([cmd_header, set_rw_cmd, cmd_footer]) - logging.info("Clear readonly bit on 'Disk%s'" % did) - session.cmd(set_rw_cmd) - - if re.search(r"Volume.*%s" % fstype, details, re.I | re.M) and not force: - logging.info("Disk%s has been formated, cancel format" % did) - continue - - if not size: - mkpart_cmd = 'echo create partition primary >> disk' - else: - mkpart_cmd = 'echo create partition primary size=%s ' - mkpart_cmd += '>> disk' - mkpart_cmd = mkpart_cmd % size - list_cmd = ' && echo list partition >> disk ' - cmds = ' '.join([cmd_header, mkpart_cmd, list_cmd, cmd_footer]) - logging.info("Create partition on 'Disk%s'" % did) - partition_index = re.search( - r'\*\s+Partition\s(\d+)\s+', session.cmd(cmds), re.M).group(1) - logging.info("Format the 'Disk%s' to %s" % (did, fstype)) - format_cmd = 'echo list partition >> disk && ' - format_cmd += 'echo select partition %s >> disk && ' % partition_index - if not mountpoint: - format_cmd += 'echo assign >> disk && ' - else: - format_cmd += 'echo assign letter=%s >> disk && ' % mountpoint - format_cmd += 'echo format fs=%s quick >> disk ' % fstype - format_cmd = ' '.join([cmd_header, format_cmd, cmd_footer]) - session.cmd(format_cmd, timeout=300) - - return True - - return False - - -def format_linux_disk(session, did, all_disks_did, partition=False, - mountpoint=None, size=None, fstype="ext3"): - """ - Create a partition on disk in linux guest and format and mount it. - - :param session: session object to guest. - :param did: disk kname, serial or wwn. - :param all_disks_did: all disks did lists each include - disk kname, serial and wwn. - :param partition: if true, can format all disks; otherwise, - only format the ones with no partition originally. - :param mountpoint: mount point for the disk. - :param size: partition size. - :param fstype: filesystem type for the disk. - :return Boolean: disk usable or not. - """ - disks = list_linux_guest_disks(session, partition) - logging.debug("Disks detail: %s" % disks) - for line in disks: - kname = line.split('/')[2] - did_list = all_disks_did[kname] - if did not in did_list: - # Continue to search target disk - continue - if not size: - size_output = session.cmd_output_safe("lsblk -oKNAME,SIZE|grep %s" - % kname) - size = size_output.splitlines()[0].split()[1] - all_disks_before = list_linux_guest_disks(session, True) - devname = line - logging.info("Create partition on disk '%s'" % devname) - mkpart_cmd = "parted -s %s mklabel gpt mkpart " - mkpart_cmd += "primary 0 %s" - mkpart_cmd = mkpart_cmd % (devname, size) - session.cmd_output_safe(mkpart_cmd) - session.cmd_output_safe("partprobe %s" % devname) - all_disks_after = list_linux_guest_disks(session, True) - partname = (all_disks_after - all_disks_before).pop() - logging.info("Format partition to '%s'" % fstype) - format_cmd = "yes|mkfs -t %s %s" % (fstype, partname) - session.cmd_output_safe(format_cmd) - if not mountpoint: - session.cmd_output_safe("mkdir /mnt/%s" % kname) - mountpoint = os.path.join("/mnt", kname) - logging.info("Mount the disk to '%s'" % mountpoint) - mount_cmd = "mount -t %s %s %s" % (fstype, partname, mountpoint) - session.cmd_output_safe(mount_cmd) - return True - - return False - - -def format_guest_disk(session, did, all_disks_did, ostype, partition=False, - mountpoint=None, size=None, fstype=None): - """ - Create a partition on disk in guest and format and mount it. - - :param session: session object to guest. - :param did: disk ID in guest. - :param all_disks_did: a dict contains all disks did lists each - include disk kname, serial and wwn for linux guest. - :param ostype: guest os type 'windows' or 'linux'. - :param partition: if true, can format all disks; otherwise, - only format the ones with no partition originally. - :param mountpoint: mount point for the disk. - :param size: partition size. - :param fstype: filesystem type for the disk; when it's the default None, - it will use the default one for corresponding ostype guest - :return Boolean: disk usable or not. - """ - default_fstype = "ntfs" if (ostype == "windows") else "ext3" - fstype = fstype or default_fstype - if ostype == "windows": - return format_windows_disk(session, did, mountpoint, size, fstype) - return format_linux_disk(session, did, all_disks_did, partition, - mountpoint, size, fstype) - - def get_linux_drive_path(session, did, timeout=120): """ Get drive path in guest by drive serial or wwn diff --git a/virttest/utils_v2v.py b/virttest/utils_v2v.py index fe244915647..f9f3dcd622a 100644 --- a/virttest/utils_v2v.py +++ b/virttest/utils_v2v.py @@ -11,6 +11,7 @@ from avocado.utils import path from avocado.utils import process +from avocado.utils import wait from avocado.core import exceptions from virttest.compat_52lts import results_stdout_52lts, results_stderr_52lts, decode_to_text @@ -23,6 +24,7 @@ from virttest import remote from virttest import utils_misc from virttest import ssh_key +from virttest import utils_disk try: V2V_EXEC = path.find_command('virt-v2v') @@ -106,7 +108,7 @@ def cleanup(self): Cleanup NFS mount records """ for src, dst, fstype in self.mount_records.values(): - utils_misc.umount(src, dst, fstype) + utils_disk.umount(src, dst, fstype) self.cleanup_authorized_keys_in_vmx() @@ -201,6 +203,17 @@ def _compose_input_transport_options(): mount_point = v2v_mount( self.vddk_libdir_src, 'vddk_libdir') + if not os.path.exists(mount_point): + os.makedirs(mount_point) + + if not utils_disk.mount( + self.vddk_libdir_src, + mount_point, + 'nfs', + verbose=True): + raise exceptions.TestError( + 'Mount %s for %s failed' % + (self.vddk_libdir_src, mount_point)) self.vddk_libdir = mount_point self.mount_records[len(self.mount_records)] = ( self.vddk_libdir_src, self.vddk_libdir, None) @@ -523,7 +536,7 @@ def wait_for_x_start(self, timeout=30): cmd = 'xset -q' if self.run_cmd(cmd)[0] == 127: return - utils_misc.wait_for( + wait.wait_for( lambda: not bool( self.run_cmd( cmd, @@ -1088,7 +1101,7 @@ def v2v_mount(src, dst='v2v_mount_point'): if not os.path.exists(mount_point): os.makedirs(mount_point) - if not utils_misc.mount( + if not utils_disk.mount( src, mount_point, 'nfs',