Merge pull request #39 from bitcraze/toverumar/ignore_tests

Ignore tests instead of skipping them if no devices for it

README.md

@@ -12,13 +12,21 @@ Before running the test suite you need to define a site in the `sites/` folder.
 The site `TOML` tells the test suite which devices to tests, what capabilities
 and decks they have, and how to reach them.
 
+The default site will be single-cf
+
 See [site docmentation](docs/development/sites.md) for the site file format to define new test sites.
 ## Running the test
 
 To run the test for a single Crazyflie, run:
 ```
-CRAZY_SITE=single-cf pytest --verbose tests/QA
+CRAZY_SITE=single-cf pytest --verbose tests/QA -k test_filter
+```
+
+or specify the name(s) of crazyflies to run on
 ```
+CRAZY_DEVICE=cf21_flow2_multiranger,cf21_flow2... pytest --verbose tests/QA -k test_filter
+```
+
 
 If you have defined your own site, then change the `CRAZY_SITE` environment
 variable to reflect that. For more information see the [running tests documentation](docs/usetests.md).

conftest.py

@@ -24,24 +24,38 @@
 DIR = os.path.dirname(os.path.realpath(__file__))
 SITE_PATH = os.path.join(DIR, 'sites/')
 REQUIREMENT = os.path.join(DIR, 'requirements/')
-
+DEFAULT_SITE = 'single-cf'
 
 USB_Power_Control = namedtuple('Port', ['hub', 'port'])
 
 
 def pytest_generate_tests(metafunc):
     has_decks = metafunc.definition.get_closest_marker('decks')
     has_properties = metafunc.definition.get_closest_marker('requirements')
+    exclude_decks = metafunc.definition.get_closest_marker('exclude_decks')
     has_decks = has_decks.args if has_decks else []
     has_properties = has_properties.args if has_properties else []
-
-    devices = get_devices(has_decks,has_properties)
+    exclude_decks = exclude_decks.args if exclude_decks else []
+    devices = get_devices(has_decks,has_properties, exclude_decks)
     if devices:
         param_name = 'test_setup' if 'test_setup' in metafunc.fixturenames else 'dev'
         metafunc.parametrize(param_name, devices, indirect=True if param_name == 'test_setup' else False, ids=lambda d: d.name)
     else:
         print(f'No devices found for test {metafunc.definition.name}')
-        metafunc.parametrize("test_setup", [pytest.param(None, marks=pytest.mark.skip(reason="No device for test"))]) #This is a bit overly complicated but pytest.skip will skip all tests in module
+        metafunc.parametrize("test_setup", [pytest.param(None, marks=pytest.mark.ignore(reason="No device for test"))]) #This is a bit overly complicated but pytest.skip will skip all tests in module
+
+
+def pytest_collection_modifyitems(config, items):
+
+    selected = list(items)
+    deselected = []
+    for test_item in items:
+        if test_item.get_closest_marker('ignore'):
+            selected.remove(test_item)
+            deselected.append(test_item)
+
+    items[:] = selected
+    config.hook.pytest_deselected(items=deselected)
 
 class USB_Power_Control_Action(str, Enum):
     ON     = 'on'
@@ -273,10 +287,11 @@ def get_bl_address(dev: BCDevice) -> str:
     link.close()
     return address
 
-def get_devices(has_decks: List[str]=[], has_properties: List[str]=[]) -> List[BCDevice]:
+
+def get_devices(has_decks: List[str]=[], has_properties: List[str]=[], exclude_decks= []) -> List[BCDevice]:
     devices = list()
 
-    site = os.getenv('CRAZY_SITE')
+    site = os.getenv('CRAZY_SITE') or DEFAULT_SITE
     devicenames = os.getenv('CRAZY_DEVICE')
     if site is None:
         raise Exception('No CRAZY_SITE env specified!')
@@ -288,11 +303,15 @@ def get_devices(has_decks: List[str]=[], has_properties: List[str]=[]) -> List[B
         site_t = toml.load(open(path, 'r'))
 
         for name, device in site_t['device'].items():
-            if(not devicenames or name in devicenames):
-                dev = BCDevice(name, device)
-                if all(deck in dev.decks for deck in has_decks):
-                    if all(prop in dev.properties for prop in has_properties):
-                        devices.append(dev)
+            dev = BCDevice(name, device)
+            conditions = [
+                (not devicenames or name in devicenames),
+                all(deck in dev.decks for deck in has_decks),
+                all(prop in dev.properties for prop in has_properties),
+                all(deck not in dev.decks for deck in exclude_decks)
+            ]
+            if all(conditions):
+                    devices.append(dev)
     except Exception:
         raise Exception(f'Failed to parse toml {path}!')
     return devices

docs/development/fixtures.md

@@ -52,7 +52,8 @@ class TestParameters:
 The fixture concept of **parametrize** basicly means **run the test for all the combinations we create here**. So, what this translates to is that we want to run the tests in the class `TestParameters` for all the devices returned by `conftest.get_devices()`. The `indirect=True` parameter means that we want to *preprocess* the device before we pass it as an argument to test. We pass it to the `test_setup` function above. The steps outlined are, for each test:
 
 1. Get the list of devices defined in a [site](sites.md)
-2. For each device in the list ...
+2. Get the properties and decks required for the test and check if the deck has them. If not, do not generate a test
+3. For each device in the list ...
    * ... pass device as an argument to `test_setup` and get a `DeviceFixure` class ...
    * ... run the test methods with `DeviceFixture` as a fixture ...
    * ... which means we will get the class as an argument and that the code after `yield` will be run when each test is complete

docs/development/sites.md

@@ -37,6 +37,9 @@ radio = "radio://0/50/2M/E7E7E71706"
 # A device _can_ have decks, could be empty
 decks = ["bcFlow2", "bcLighthouse4"]
 
+# A device can have properties, could be empty
+properties = ["powerMeasurement", "wifi"]
+
 # A device _may_ have a bootloader URI, if it has we can use it to try to recover
 # from bootloader mode.
 bootloader_radio = "radio://0/0/2M/B19CF77F05?safelink=0"

pytest.ini

@@ -7,3 +7,5 @@ junit_log_passing_tests = 1
 markers = 
     decks: decks that a test needs to run
     requirements: other requirements that a test needs to run
+    exclude_decks: decks that makes a test not runnable
+    ignore: ignore a test

tests/QA/test_log.py

@@ -127,14 +127,13 @@ def test_log_too_much_per_block(self, test_setup: conftest.DeviceFixture):
         with pytest.raises(AttributeError):
             test_setup.device.cf.log.add_config(config)
 
+    @pytest.mark.exclude_decks('bcDWM1000','bcFlow', 'bcFlow2', 'lighthouse4')
     def test_log_stress(self, test_setup: conftest.DeviceFixture):
         '''
         Make sure we can receive all packets requested when having an effective
         rate of logging.rate packets/s.
         '''
         requirement = conftest.get_requirement('logging.rate')
-        if test_setup.kalman_active:
-            pytest.skip('Only on non-kalman')
 
         configs = []
         duration = 10.0