twister: harness_config extended by test retries in testcase.yaml

doc/develop/test/twister.rst

@@ -611,6 +611,18 @@ harness_config: <harness configuration options>
         Only one fixture can be defined per test scenario and the fixture name has to
         be unique across all tests in the test suite.
 
+    suite_repeat: <int> (default 1)
+        This parameter specifies the number of times the entire test suite should be repeated.
+
+    test_repeat: <int> (default 1)
+        This parameter specifies the number of times each individual test within the test suite
+        should be repeated.
+
+    test_shuffle: <True|False> (default False)
+        This parameter indicates whether the order of the tests within the test suite should
+        be shuffled. When set to ``true``, the tests will be executed in a random order.
+
+
 .. _pytest_root:
 
     pytest_root: <list of pytest testpaths> (default pytest)

scripts/pylib/twister/twisterlib/testinstance.py

@@ -60,6 +60,9 @@ def __init__(self, testsuite, platform, outdir):
         self.execution_time = 0
         self.build_time = 0
         self.retries = 0
+        self.suite_repeat = None
+        self.test_repeat = None
+        self.test_shuffle = False
 
         self.name = os.path.join(platform.name, testsuite.name)
         self.dut = None
@@ -322,6 +325,23 @@ def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable
 
             content = "\n".join(new_config_list)
 
+
+        if self.testsuite.harness_config:
+            self.suite_repeat = self.testsuite.harness_config.get('ztest_suite_repeat', None)
+            self.test_repeat = self.testsuite.harness_config.get('ztest_test_repeat', None)
+            self.test_shuffle = self.testsuite.harness_config.get('ztest_test_shuffle', False)
+
+
+        # Use suite_repeat and test_repeat values
+        if self.suite_repeat or self.test_repeat or self.test_shuffle:
+            content +="\nCONFIG_ZTEST_REPEAT=y"
+            if self.suite_repeat:
+                content += f"\nCONFIG_ZTEST_SUITE_REPEAT_COUNT={self.suite_repeat}"
+            if self.test_repeat:
+                content += f"\nCONFIG_ZTEST_TEST_REPEAT_COUNT={self.test_repeat}"
+            if self.test_shuffle:
+                content += f"\nCONFIG_ZTEST_SHUFFLE=y"
+
         if enable_coverage:
             for cp in coverage_platform:
                 if cp in platform.aliases:

scripts/schemas/twister/testsuite-schema.yaml

@@ -141,6 +141,15 @@ schema;scenario-schema:
         "bsim_exe_name":
           type: str
           required: false
+        "ztest_suite_repeat":
+          type: int
+          required: false
+        "ztest_test_repeat":
+          type: int
+          required: false
+        "ztest_test_shuffle":
+          type: bool
+          required: false
     "min_ram":
       type: int
       required: false

scripts/tests/twister/test_testinstance.py

@@ -649,3 +649,38 @@ def test_testinstance_get_buildlog_file(tmp_path, testinstance, create_build_log
 
     if expected_error is None:
         assert res == str(build_log)
+
+
+TESTDATA_9 = [
+    (
+        {'ztest_suite_repeat': 5, 'ztest_test_repeat': 10, 'ztest_test_shuffle': True},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=5\nCONFIG_ZTEST_TEST_REPEAT_COUNT=10\nCONFIG_ZTEST_SHUFFLE=y'
+    ),
+    (
+        {'ztest_suite_repeat': 3},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=3'
+    ),
+    (
+        {'ztest_test_repeat': 7},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_TEST_REPEAT_COUNT=7'
+    ),
+    (
+        {'ztest_test_shuffle': True},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SHUFFLE=y'
+    ),
+    (
+        {},
+        ''
+    ),
+]
+
+@pytest.mark.parametrize('harness_config, expected_content', TESTDATA_9)
+def test_create_overlay_with_harness_config(class_testplan, all_testsuites_dict, platforms_list, harness_config, expected_content):
+    testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app'
+    class_testplan.testsuites = all_testsuites_dict
+    testsuite = class_testplan.testsuites.get(testsuite_path)
+    testsuite.harness_config = harness_config
+    class_testplan.platforms = platforms_list
+    platform = class_testplan.get_platform("demo_board_2")
+    testinstance = TestInstance(testsuite, platform, class_testplan.env.outdir)
+    assert testinstance.create_overlay(platform) == expected_content

tests/ztest/repeat/CMakeLists.txt

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(repeat)
+
+target_sources(app PRIVATE src/main.c)

tests/ztest/repeat/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

tests/ztest/repeat/src/main.c

@@ -0,0 +1,30 @@
+#include <zephyr/ztest.h>
+
+extern struct ztest_suite_stats UTIL_CAT(z_ztest_suite_node_stats_, testsuite);
+struct ztest_suite_stats *suite_stats = &UTIL_CAT(z_ztest_suite_node_stats_, testsuite);
+extern struct ztest_unit_test_stats z_ztest_unit_test_stats_testsuite_test_repeating1;
+struct ztest_unit_test_stats *case_stats = &z_ztest_unit_test_stats_testsuite_test_repeating1;
+
+ZTEST(testsuite, test_repeating1)
+{
+	ztest_test_pass();
+}
+
+ZTEST(testsuite, test_repeating2)
+{
+	ztest_test_pass();
+}
+
+ZTEST(testsuite, test_repeating3)
+{
+	ztest_test_pass();
+}
+
+static void repeat_teardown(void *)
+{
+	/* run_count + 1 counter is incremented after the testcase is executed. */
+	printk("Test suite executed: %d times.\n", suite_stats->run_count + 1);
+	printk("Test case executed : %d times.\n", case_stats->run_count);
+}
+
+ZTEST_SUITE(testsuite, NULL, NULL, NULL, NULL, repeat_teardown);

tests/ztest/repeat/testcase.yaml

@@ -0,0 +1,63 @@
+tests:
+  testing.ztest.repeat_suite_regex:
+    platform_allow: qemu_x86
+    harness: console
+    harness_config:
+      ztest_suite_repeat: 3
+      ztest_test_repeat: 2
+      type: multi_line
+      regex:
+        - "Running TESTSUITE testsuite"
+        - "Running TESTSUITE testsuite"
+        - "Running TESTSUITE testsuite"
+
+  testing.ztest.repeat_testcase_regex:
+    platform_allow: qemu_x86
+    harness: console
+    harness_config:
+      ztest_suite_repeat: 1
+      ztest_test_repeat: 5
+      type: multi_line
+      regex:
+        - "START - test_repeating1"
+        - "START - test_repeating1"
+        - "START - test_repeating1"
+        - "START - test_repeating1"
+        - "START - test_repeating1"
+
+  testing.ztest.repeat_no_shuffle_regex:
+    platform_allow: qemu_x86
+    harness: console
+    harness_config:
+      ztest_suite_repeat: 1
+      ztest_test_repeat: 3
+      type: multi_line
+      regex:
+        - "START - test_repeating1"
+        - "START - test_repeating2"
+        - "START - test_repeating3"
+        - "START - test_repeating1"
+        - "START - test_repeating2"
+        - "START - test_repeating3"
+        - "START - test_repeating1"
+        - "START - test_repeating2"
+        - "START - test_repeating3"
+
+  testing.ztest.repeat_shuffle_regex:
+    platform_allow: qemu_x86
+    harness: console
+    harness_config:
+      ztest_suite_repeat: 1
+      ztest_test_repeat: 3
+      ztest_test_shuffle: true
+      type: multi_line
+      regex:
+        - "START - test_repeating2"
+        - "START - test_repeating1"
+        - "START - test_repeating3"
+        - "START - test_repeating2"
+        - "START - test_repeating1"
+        - "START - test_repeating3"
+        - "START - test_repeating2"
+        - "START - test_repeating3"
+        - "START - test_repeating1"