diff --git a/doc/develop/test/twister.rst b/doc/develop/test/twister.rst index f535648dd6e91..52cdac9622352 100644 --- a/doc/develop/test/twister.rst +++ b/doc/develop/test/twister.rst @@ -611,6 +611,18 @@ harness_config: Only one fixture can be defined per test scenario and the fixture name has to be unique across all tests in the test suite. + suite_repeat: (default 1) + This parameter specifies the number of times the entire test suite should be repeated. + + test_repeat: (default 1) + This parameter specifies the number of times each individual test within the test suite + should be repeated. + + test_shuffle: (default False) + This parameter indicates whether the order of the tests within the test suite should + be shuffled. When set to ``true``, the tests will be executed in a random order. + + .. _pytest_root: pytest_root: (default pytest) diff --git a/scripts/pylib/twister/twisterlib/testinstance.py b/scripts/pylib/twister/twisterlib/testinstance.py index 825403e8d84c3..1e92223b7fd08 100644 --- a/scripts/pylib/twister/twisterlib/testinstance.py +++ b/scripts/pylib/twister/twisterlib/testinstance.py @@ -60,6 +60,9 @@ def __init__(self, testsuite, platform, outdir): self.execution_time = 0 self.build_time = 0 self.retries = 0 + self.suite_repeat = None + self.test_repeat = None + self.test_shuffle = False self.name = os.path.join(platform.name, testsuite.name) self.dut = None @@ -322,6 +325,23 @@ def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable content = "\n".join(new_config_list) + + if self.testsuite.harness_config: + self.suite_repeat = self.testsuite.harness_config.get('ztest_suite_repeat', None) + self.test_repeat = self.testsuite.harness_config.get('ztest_test_repeat', None) + self.test_shuffle = self.testsuite.harness_config.get('ztest_test_shuffle', False) + + + # Use suite_repeat and test_repeat values + if self.suite_repeat or self.test_repeat or self.test_shuffle: + content +="\nCONFIG_ZTEST_REPEAT=y" + if self.suite_repeat: + content += f"\nCONFIG_ZTEST_SUITE_REPEAT_COUNT={self.suite_repeat}" + if self.test_repeat: + content += f"\nCONFIG_ZTEST_TEST_REPEAT_COUNT={self.test_repeat}" + if self.test_shuffle: + content += f"\nCONFIG_ZTEST_SHUFFLE=y" + if enable_coverage: for cp in coverage_platform: if cp in platform.aliases: diff --git a/scripts/schemas/twister/testsuite-schema.yaml b/scripts/schemas/twister/testsuite-schema.yaml index 4cb81431d86b8..0cb10ff9c4c19 100644 --- a/scripts/schemas/twister/testsuite-schema.yaml +++ b/scripts/schemas/twister/testsuite-schema.yaml @@ -141,6 +141,15 @@ schema;scenario-schema: "bsim_exe_name": type: str required: false + "ztest_suite_repeat": + type: int + required: false + "ztest_test_repeat": + type: int + required: false + "ztest_test_shuffle": + type: bool + required: false "min_ram": type: int required: false diff --git a/scripts/tests/twister/test_testinstance.py b/scripts/tests/twister/test_testinstance.py index 59ea56a207eef..07e1c35f88e27 100644 --- a/scripts/tests/twister/test_testinstance.py +++ b/scripts/tests/twister/test_testinstance.py @@ -649,3 +649,38 @@ def test_testinstance_get_buildlog_file(tmp_path, testinstance, create_build_log if expected_error is None: assert res == str(build_log) + + +TESTDATA_9 = [ + ( + {'ztest_suite_repeat': 5, 'ztest_test_repeat': 10, 'ztest_test_shuffle': True}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=5\nCONFIG_ZTEST_TEST_REPEAT_COUNT=10\nCONFIG_ZTEST_SHUFFLE=y' + ), + ( + {'ztest_suite_repeat': 3}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=3' + ), + ( + {'ztest_test_repeat': 7}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_TEST_REPEAT_COUNT=7' + ), + ( + {'ztest_test_shuffle': True}, + '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SHUFFLE=y' + ), + ( + {}, + '' + ), +] + +@pytest.mark.parametrize('harness_config, expected_content', TESTDATA_9) +def test_create_overlay_with_harness_config(class_testplan, all_testsuites_dict, platforms_list, harness_config, expected_content): + testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app' + class_testplan.testsuites = all_testsuites_dict + testsuite = class_testplan.testsuites.get(testsuite_path) + testsuite.harness_config = harness_config + class_testplan.platforms = platforms_list + platform = class_testplan.get_platform("demo_board_2") + testinstance = TestInstance(testsuite, platform, class_testplan.env.outdir) + assert testinstance.create_overlay(platform) == expected_content diff --git a/tests/ztest/repeat/CMakeLists.txt b/tests/ztest/repeat/CMakeLists.txt new file mode 100644 index 0000000000000..54523fef0fa9f --- /dev/null +++ b/tests/ztest/repeat/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(repeat) + +target_sources(app PRIVATE src/main.c) diff --git a/tests/ztest/repeat/prj.conf b/tests/ztest/repeat/prj.conf new file mode 100644 index 0000000000000..9467c2926896d --- /dev/null +++ b/tests/ztest/repeat/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/tests/ztest/repeat/src/main.c b/tests/ztest/repeat/src/main.c new file mode 100644 index 0000000000000..a9026f5101e84 --- /dev/null +++ b/tests/ztest/repeat/src/main.c @@ -0,0 +1,30 @@ +#include + +extern struct ztest_suite_stats UTIL_CAT(z_ztest_suite_node_stats_, testsuite); +struct ztest_suite_stats *suite_stats = &UTIL_CAT(z_ztest_suite_node_stats_, testsuite); +extern struct ztest_unit_test_stats z_ztest_unit_test_stats_testsuite_test_repeating1; +struct ztest_unit_test_stats *case_stats = &z_ztest_unit_test_stats_testsuite_test_repeating1; + +ZTEST(testsuite, test_repeating1) +{ + ztest_test_pass(); +} + +ZTEST(testsuite, test_repeating2) +{ + ztest_test_pass(); +} + +ZTEST(testsuite, test_repeating3) +{ + ztest_test_pass(); +} + +static void repeat_teardown(void *) +{ + /* run_count + 1 counter is incremented after the testcase is executed. */ + printk("Test suite executed: %d times.\n", suite_stats->run_count + 1); + printk("Test case executed : %d times.\n", case_stats->run_count); +} + +ZTEST_SUITE(testsuite, NULL, NULL, NULL, NULL, repeat_teardown); diff --git a/tests/ztest/repeat/testcase.yaml b/tests/ztest/repeat/testcase.yaml new file mode 100644 index 0000000000000..c092c079bd4f4 --- /dev/null +++ b/tests/ztest/repeat/testcase.yaml @@ -0,0 +1,63 @@ +tests: + testing.ztest.repeat_suite_regex: + platform_allow: qemu_x86 + harness: console + harness_config: + ztest_suite_repeat: 3 + ztest_test_repeat: 2 + type: multi_line + regex: + - "Running TESTSUITE testsuite" + - "Running TESTSUITE testsuite" + - "Running TESTSUITE testsuite" + + testing.ztest.repeat_testcase_regex: + platform_allow: qemu_x86 + harness: console + harness_config: + ztest_suite_repeat: 1 + ztest_test_repeat: 5 + type: multi_line + regex: + - "START - test_repeating1" + - "START - test_repeating1" + - "START - test_repeating1" + - "START - test_repeating1" + - "START - test_repeating1" + + testing.ztest.repeat_no_shuffle_regex: + platform_allow: qemu_x86 + harness: console + harness_config: + ztest_suite_repeat: 1 + ztest_test_repeat: 3 + type: multi_line + regex: + - "START - test_repeating1" + - "START - test_repeating2" + - "START - test_repeating3" + - "START - test_repeating1" + - "START - test_repeating2" + - "START - test_repeating3" + - "START - test_repeating1" + - "START - test_repeating2" + - "START - test_repeating3" + + testing.ztest.repeat_shuffle_regex: + platform_allow: qemu_x86 + harness: console + harness_config: + ztest_suite_repeat: 1 + ztest_test_repeat: 3 + ztest_test_shuffle: true + type: multi_line + regex: + - "START - test_repeating2" + - "START - test_repeating1" + - "START - test_repeating3" + - "START - test_repeating2" + - "START - test_repeating1" + - "START - test_repeating3" + - "START - test_repeating2" + - "START - test_repeating3" + - "START - test_repeating1"