From ef73b5aad376d0a8e6e6e6a1c357e49c070f81e8 Mon Sep 17 00:00:00 2001 From: Sebastian Mitterle Date: Mon, 21 Oct 2019 16:48:55 +0200 Subject: [PATCH] Add kernel module handler for kvm hugepage support Handle cases where kvm module is loaded without hugepage support 1. Add class to help reload and restore Linux kernel module configuration. KernelModuleHandler will only reload if current module configuration doesn't contain some of the requested parameter settings. It will only restore backed up configuration if it changed the current module configuration. 2. Hook activating hugepages with config into env_process a. Introduce variable `kvm_module_force_load` to force loading the module always. Don't force by default. b. Introduce variable `kvm_module_parameters` to define custom module parameters on load. c. Use convenience method `reload` to delegate decision if to load to new python module. d. Sample log output i. where reload is not necessary ```bash 2020-02-20 14:44:31,300 utils_kernel_mod L0089 DEBUG| Backed up kvm module state (was_loaded, params)=(True, halt_poll_ns_shrink=0 hpage=1 halt_poll_ns=50000 halt_poll_ns_grow_start=10000 halt_poll_ns_grow=2 nested=0 halt_poll_max_steal=10) 2020-02-20 14:44:31,301 utils_kernel_mod L0038 DEBUG| Not reloading module. Current module configuration for kvm already contains all requested parameters. Requested: hpage=1. Current: halt_poll_ns_shrink=0 hpage=1 halt_poll_ns=50000 halt_poll_ns_grow_start=10000 halt_poll_ns_grow=2 nested=0 halt_poll_max_steal=10. ``` ii. where reload is necessary ```bash 2020-02-20 14:30:16,509 utils_kernel_mod L0089 DEBUG| Backed up kvm module state (was_loaded, params)=(True, halt_poll_ns_shrink=0 hpage=0 halt_poll_ns=50000 halt_poll_ns_grow_start=10000 halt_poll_ns_grow=2 nested=0 halt_poll_max_steal=10) 2020-02-20 14:30:16,510 utils_kernel_mod L0049 DEBUG| Loading module: rmmod kvm; modprobe kvm hpage=1 2020-02-20 14:35:13,839 utils_kernel_mod L0069 DEBUG| Restoring module state: rmmod kvm; modprobe kvm halt_poll_ns_shrink=0 hpage=0 halt_poll_ns=50000 halt_poll_ns_grow_start=10000 halt_poll_ns_grow=2 nested=0 halt_poll_max_steal=10 ``` Signed-off-by: Sebastian Mitterle Signed-off-by: Lukas Doktor Signed-off-by: Sebastian Mitterle --- virttest/env_process.py | 11 + .../unittests/test_utils_kernel_module.py | 206 ++++++++++++++++++ virttest/utils_kernel_module.py | 196 +++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 virttest/unittests/test_utils_kernel_module.py create mode 100644 virttest/utils_kernel_module.py diff --git a/virttest/env_process.py b/virttest/env_process.py index 8ddcf93c3b..cb89ce7192 100644 --- a/virttest/env_process.py +++ b/virttest/env_process.py @@ -48,6 +48,7 @@ from virttest import utils_package from virttest import utils_qemu from virttest import migration +from virttest import utils_kernel_module from virttest.utils_version import VersionInterval from virttest.compat_52lts import decode_to_text from virttest.staging import service @@ -80,6 +81,8 @@ postprocess_vm_on_hook = None postprocess_vm_off_hook = None +#: Object to handle kvm module reloads with certain parameters +KVM_MODULE_HANDLER = None #: QEMU version regex. Attempts to extract the simple and extended version #: information from the output produced by `qemu -version` @@ -1049,6 +1052,11 @@ def preprocess(test, params, env): env["cpu_driver"] = cpu_driver[0] params["cpu_driver"] = env.get("cpu_driver") + kvm_module_params = params.get("kvm_module_parameters", "") + force_load = params.get("kvm_module_force_load", "no") == "yes" + global KVM_MODULE_HANDLER + KVM_MODULE_HANDLER = utils_kernel_module.reload("kvm", force_load, kvm_module_params) + version_info = {} # Get the KVM kernel module version if os.path.exists("/dev/kvm"): @@ -1589,6 +1597,9 @@ def postprocess(test, params, env): err += "\nTHP cleanup: %s" % str(details).replace('\\n', '\n ') logging.error(details) + if KVM_MODULE_HANDLER: + KVM_MODULE_HANDLER.restore() + if params.get("setup_ksm") == "yes": try: ksm = test_setup.KSMConfig(params, env) diff --git a/virttest/unittests/test_utils_kernel_module.py b/virttest/unittests/test_utils_kernel_module.py new file mode 100644 index 0000000000..4546976a05 --- /dev/null +++ b/virttest/unittests/test_utils_kernel_module.py @@ -0,0 +1,206 @@ +import unittest + +try: + from unittest import mock +except ImportError: + import mock + +from virttest import utils_kernel_module + +# Test values +some_module_name = 'skvm' +some_module_param = 'force_emulation_prefix' +some_module_val = 'N' +some_module_config = {some_module_param: some_module_val, + 'halt_poll_ns_shrink': '0'} +some_module_params = "%s=%s" % (some_module_param, some_module_val) + +# Mocks +getstatusoutput_ok = mock.Mock(return_value=(0, "")) + + +@mock.patch.object(utils_kernel_module.os, 'listdir', return_value=[some_module_param]) +@mock.patch.object(utils_kernel_module, 'open', mock.mock_open(read_data=some_module_val + '\n')) +@mock.patch.object(utils_kernel_module.process, 'getstatusoutput', getstatusoutput_ok) +class TestReloadModule(unittest.TestCase): + """ + Tests the reload_module method + """ + + def tearDown(self): + getstatusoutput_ok.reset_mock() + + def assertLoadedWith(self, params): + self.assertTrue(getstatusoutput_ok.called) + cmd = getstatusoutput_ok.call_args[0][0] + if params != "": + self.assertTrue(params in cmd) + else: + self.assertTrue(cmd.endswith(some_module_name)) + + def assertNotLoaded(self): + getstatusoutput_ok.assert_not_called() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=False) + def test_tc1(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=True, params=some_module_params) + self.assertLoadedWith(some_module_params) + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=False) + def test_tc2(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=False, params=some_module_params) + self.assertNotLoaded() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc3(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=True, params=some_module_params) + self.assertLoadedWith(some_module_params) + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc3_empty(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=True, params="") + self.assertLoadedWith("") + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc4(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=True, params="key=value") + self.assertLoadedWith("key=value") + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc5(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=False, params=some_module_params) + self.assertNotLoaded() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc5_empty(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=False, params="") + self.assertNotLoaded() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc6(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(force=False, params="key=value") + self.assertLoadedWith("key=value") + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc6_partial(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.assertEqual(some_module_params, self.handler.config_backup) + self.handler.reload_module(force=False, params=some_module_params + " key=value") + self.assertLoadedWith("key=value") + + +@mock.patch.object(utils_kernel_module.process, 'getstatusoutput', getstatusoutput_ok) +@mock.patch.object(utils_kernel_module.os, 'listdir', return_value=[some_module_param]) +@mock.patch.object(utils_kernel_module, 'open', mock.mock_open(read_data=some_module_val + '\n')) +class TestInit(unittest.TestCase): + """ + Tests if module status is backed up correctly + """ + + def tearDown(self): + getstatusoutput_ok.reset_mock() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_reloaded(self, *mocks): + handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.assertTrue(handler.was_loaded) + self.assertEqual(some_module_params, handler.config_backup) + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=False) + def test_not_reloaded(self, *mocks): + handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.assertFalse(handler.was_loaded) + self.assertIsNone(handler.config_backup) + + +@mock.patch.object(utils_kernel_module.os, 'listdir', return_value=[some_module_param]) +@mock.patch.object(utils_kernel_module, 'open', mock.mock_open(read_data=some_module_val + '\n')) +@mock.patch.object(utils_kernel_module.process, 'getstatusoutput', getstatusoutput_ok) +class TestRestore(unittest.TestCase): + """ + Tests the restore method + """ + + def tearDown(self): + getstatusoutput_ok.reset_mock() + + def assertRestored(self, params): + self.assertTrue(getstatusoutput_ok.called) + cmd = getstatusoutput_ok.call_args[0][0] + self.assertTrue(params in cmd) + + def assertNoRestore(self): + self.assertFalse(getstatusoutput_ok.called) + + def assertUnloaded(self): + self.assertTrue(getstatusoutput_ok.called) + cmd = getstatusoutput_ok.call_args[0][0] + self.assertTrue("modprobe -r" in cmd) + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc1(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + orig_config = self.handler.config_backup + self.handler.reload_module(True, "key=value") + self.handler.restore() + self.assertRestored(orig_config) + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc1_reload_twice(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + orig_config = self.handler.config_backup + self.handler.reload_module(True, "key=value") + self.handler.reload_module(True, "key1=value1") + self.handler.restore() + self.assertRestored(orig_config) + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=True) + def test_tc2(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.restore() + self.assertNoRestore() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=False) + def test_tc3(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.reload_module(True, "key=value") + self.handler.restore() + self.assertUnloaded() + + @mock.patch.object(utils_kernel_module.os.path, 'exists', return_value=False) + def test_tc4(self, *mocks): + self.handler = utils_kernel_module.KernelModuleHandler(some_module_name) + self.handler.restore() + self.assertNoRestore() + + +@mock.patch.object(utils_kernel_module.KernelModuleHandler, '__init__', return_value=None) +@mock.patch.object(utils_kernel_module.KernelModuleHandler, 'reload_module', return_value=None) +class TestReload(unittest.TestCase): + """ + Tests the module global reload method + """ + + def test_empty_params_1(self, *mocks): + self.assertIsNone(utils_kernel_module.reload(some_module_name, force=False, params="")) + + def test_empty_params_2(self, *mocks): + self.assertIsNotNone(utils_kernel_module.reload(some_module_name, force=True, params="")) + + def test_non_empty_params_1(self, *mocks): + self.assertIsNotNone(utils_kernel_module.reload(some_module_name, force=False, params=some_module_params)) + + def test_non_empty_params_2(self, *mocks): + self.assertIsNotNone(utils_kernel_module.reload(some_module_name, force=True, params=some_module_params)) + + +if __name__ == '__main__': + unittest.main() diff --git a/virttest/utils_kernel_module.py b/virttest/utils_kernel_module.py new file mode 100644 index 0000000000..64c2963ea4 --- /dev/null +++ b/virttest/utils_kernel_module.py @@ -0,0 +1,196 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2020 +# Author: Sebastian Mitterle + +""" +Handle Linux kernel modules +""" + +import os +import logging + +from avocado.utils import process +from avocado.core import exceptions + + +def reload(module_name, force, params): + """ + Convenience method that creates a KernelModuleHandler instance + and reloads the module only if any action will is required. + + :param module_name: name of kernel module to be handled + :param force: if to force load with params in any case, e.g. True + :param params: parameters to load with, e.g. 'key1=param1 ...' + :return: instance if module was loaded + """ + + if params != "" or force: + handler = KernelModuleHandler(module_name) + handler.reload_module(force, params) + return handler + return None + + +class KernelModuleHandler(object): + """Class handling Linux kernel modules""" + + def __init__(self, module_name): + """Create kernel module handler""" + + self._module_name = module_name + self._was_loaded = None + self._loaded_config = None + self._config_backup = None + self._backup_config() + + def reload_module(self, force, params): + """ + Reload module with given parameters. + + If force=False loading will be skipped when either the module is + already loaded with the passed parameters or when the module has + not been loaded at all. + + +----------------------+-+-+-+-+-+-+ + |**precondition** | + +----------------------+-+-+-+-+-+-+ + |module loaded |N|N|Y|Y|Y|Y| + +----------------------+-+-+-+-+-+-+ + |params already loaded |*|*|Y|N|Y|N| + +----------------------+-+-+-+-+-+-+ + |force load |Y|N|Y|Y|N|N| + +----------------------+-+-+-+-+-+-+ + |**result** | + +----------------------+-+-+-+-+-+-+ + |issue reload |Y|N|Y|Y|N|Y| + +----------------------+-+-+-+-+-+-+ + + :param force: if to force load with params in any case, e.g. True + :param params: parameters to load with, e.g. 'key1=param1 ...' + """ + + current_config = self._get_serialized_config() + if not force: + do_not_load = False + if (current_config and + all(x in current_config.split() for x in params.split())): + logging.debug("Not reloading module. Current module config" + " uration for %s already contains all reques" + " ted parameters. Requested: '%s'. Current:" + " '%s'. Use force=True to force loading.", + self._module_name, params, current_config) + do_not_load = True + elif not self._was_loaded: + logging.debug("Module %s isn't loaded. Use force=True to force" + " loading.", self._module_name) + do_not_load = True + if do_not_load: + return + + cmds = [] + if self._was_loaded: + # TODO: Handle cases were module cannot be removed + cmds.append('modprobe -r --remove-dependencies %s' % self._module_name) + cmd = 'modprobe %s %s' % (self._module_name, params) + cmds.append(cmd.strip()) + logging.debug("Loading module: %s", cmds) + for cmd in cmds: + status, output = process.getstatusoutput(cmd, ignore_status=True) + if status: + raise exceptions.TestError("Couldn't load module %s: %s" % ( + self._module_name, output + )) + self._loaded_config = params + + def restore(self): + """ + Restore previous module state. + + The state will only be restored if the original state + was altered. + + +-------------------+-+-+-+-+ + |**precondition** | + +-------------------+-+-+-+-+ + |module loaded |Y|Y|N|N| + +-------------------+-+-+-+-+ + |loaded with params |Y|N|Y|N| + +-------------------+-+-+-+-+ + |**result** | + +-------------------+-+-+-+-+ + |issue restore |Y|N|Y|N| + +-------------------+-+-+-+-+ + """ + + if self._loaded_config is not None: + cmds = [] + # TODO: Handle cases were module cannot be removed + cmds.append('modprobe -r --remove-dependencies %s' % self._module_name) + if self._was_loaded: + cmd = 'modprobe %s %s' % (self._module_name, + self._config_backup) + cmds.append(cmd.strip()) + logging.debug("Restoring module state: %s", cmds) + for cmd in cmds: + status, output = process.getstatusoutput(cmd, ignore_status=True) + if status: + raise exceptions.TestError( + "Couldn't restore module %s. Command '%s'." + " Output '%s'." + % (self._module_name, cmd, output)) + + def _backup_config(self): + """ + Check if self.module_name is loaded and read config + + """ + config = self._get_serialized_config() + if config is not None: + self._config_backup = config + self._was_loaded = True + else: + self._was_loaded = False + logging.debug("Backed up %s module state (was_loaded, params)" + "=(%s, %s)", self._module_name, self._was_loaded, + self._config_backup) + + @property + def was_loaded(self): + """ Read-only property """ + + return self._was_loaded + + @property + def config_backup(self): + """ Read-only property """ + + return self._config_backup + + def _get_serialized_config(self): + """ + Get current module parameters + + :return: String holding module config 'param1=value1 param2=value2 ...', None if + module not loaded + """ + + mod_params_path = '/sys/module/%s/parameters/' % self._module_name + if not os.path.exists(mod_params_path): + return None + + mod_params = {} + params = os.listdir(mod_params_path) + for param in params: + with open(os.path.join(mod_params_path, param), 'r') as param_file: + mod_params[param] = param_file.read().strip() + return " ".join("%s=%s" % _ for _ in mod_params.items()) if mod_params else ""