Skip to content

Commit

Permalink
Add support for randomizing test execution order
Browse files Browse the repository at this point in the history 
This commit reintroduces the option to shuffle the test execution order
into the test runner. This has been tested with the temp_sensor example
project in Ceedling. Unit tests have also been successfully executed.

Signed-off-by: James Raphael Tiovalen <[email protected]>
  • Loading branch information
@jamestiotio
jamestiotio committed Feb 11, 2024
1 parent 64939db commit 22ed956
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 7 deletions.
98 changes: 91 additions & 7 deletions auto/generate_test_runner.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ def self.default_options
use_param_tests: false,
use_system_files: true,
include_extensions: '(?:hpp|hh|H|h)',
source_extensions: '(?:cpp|cc|ino|C|c)'
source_extensions: '(?:cpp|cc|ino|C|c)',
shuffle_tests: false,
rng_seed: 0
}
end

Expand Down Expand Up @@ -90,6 +92,7 @@ def run(input_file, output_file, options = nil)
def generate(input_file, output_file, tests, used_mocks, testfile_includes)
File.open(output_file, 'w') do |output|
create_header(output, used_mocks, testfile_includes)
create_run_test_params_struct(output)
create_externs(output, tests, used_mocks)
create_mock_management(output, used_mocks)
create_setup(output)
Expand All @@ -99,6 +102,7 @@ def generate(input_file, output_file, tests, used_mocks, testfile_includes)
create_reset(output)
create_run_test(output) unless tests.empty?
create_args_wrappers(output, tests)
create_shuffle_tests(output) if @options[:shuffle_tests]
create_main(output, input_file, tests, used_mocks)
end

Expand Down Expand Up @@ -231,16 +235,40 @@ def find_setup_and_teardown(source)
@options[:has_suite_teardown] ||= (source =~ /int\s+suiteTearDown\s*\(int\s+([a-zA-Z0-9_])+\s*\)/)
end

def count_tests(tests)
if @options[:use_param_tests]
idx = 0
tests.each do |test|
if (test[:args].nil? || test[:args].empty?)
idx += 1
else
test[:args].each do |args|
idx += 1
end
end
end
return idx
else
return tests.size
end
end

def create_header(output, mocks, testfile_includes = [])
output.puts('/* AUTOGENERATED FILE. DO NOT EDIT. */')
output.puts("\n/*=======Automagically Detected Files To Include=====*/")
output.puts('extern "C" {') if @options[:externcincludes]
if @options[:shuffle_tests]
output.puts('#include <stdlib.h>')
if @options[:rng_seed] == 0
output.puts('#include <time.h>')
end
end
output.puts("#include \"#{@options[:framework]}.h\"")
output.puts('#include "cmock.h"') unless mocks.empty?
output.puts('}') if @options[:externcincludes]
if @options[:defines] && !@options[:defines].empty?
output.puts("/* injected defines for unity settings, etc */")
@options[:defines].each do |d|
@options[:defines].each do |d|
def_only = d.match(/(\w+).*/)[1]
output.puts("#ifndef #{def_only}\n#define #{d}\n#endif /* #{def_only} */")
end
Expand Down Expand Up @@ -270,6 +298,16 @@ def create_header(output, mocks, testfile_includes = [])
output.puts('char* GlobalOrderError;')
end

def create_run_test_params_struct(output)
output.puts("\n/*=======Structure Used By Test Runner=====*/")
output.puts('struct UnityRunTestParameters')
output.puts('{')
output.puts(' UnityTestFunction func;')
output.puts(' const char* name;')
output.puts(' UNITY_LINE_TYPE line_num;')
output.puts('};')
end

def create_externs(output, tests, _mocks)
output.puts("\n/*=======External Functions This Runner Calls=====*/")
output.puts("extern void #{@options[:setup_name]}(void);")
Expand Down Expand Up @@ -392,6 +430,22 @@ def create_args_wrappers(output, tests)
end
end

def create_shuffle_tests(output)
output.puts("\n/*=======Shuffle Test Order=====*/")
output.puts('static void shuffleTests(struct UnityRunTestParameters run_test_params_arr[], int num_of_tests)')
output.puts('{')

# Use Fisher-Yates shuffle algorithm
output.puts(' for (int i = num_of_tests - 1; i > 0; i--)')
output.puts(' {')
output.puts(' int j = rand() % (i + 1);')
output.puts(' struct UnityRunTestParameters temp = run_test_params_arr[i];')
output.puts(' run_test_params_arr[i] = run_test_params_arr[j];')
output.puts(' run_test_params_arr[j] = temp;')
output.puts(' }')
output.puts('}')
end

def create_main(output, filename, tests, used_mocks)
output.puts("\n/*=======MAIN=====*/")
main_name = @options[:main_name].to_sym == :auto ? "main_#{filename.gsub('.c', '')}" : (@options[:main_name]).to_s
Expand Down Expand Up @@ -437,18 +491,46 @@ def create_main(output, filename, tests, used_mocks)
else
output.puts(" UnityBegin(\"#{filename.gsub(/\\/, '\\\\\\')}\");")
end
if @options[:shuffle_tests]
output.puts
if @options[:rng_seed] == 0
output.puts(' srand(time(NULL));')
else
output.puts(" srand(#{@options[:rng_seed]});")
end
end
output.puts
output.puts(" int number_of_tests = #{count_tests(tests)};")
output.puts(' struct UnityRunTestParameters run_test_params_arr[number_of_tests];')
output.puts
idx = 0
tests.each do |test|
if (!@options[:use_param_tests]) || test[:args].nil? || test[:args].empty?
output.puts(" run_test(#{test[:test]}, \"#{test[:test]}\", #{test[:line_number]});")
output.puts(" run_test_params_arr[#{idx}].func = #{test[:test]};")
output.puts(" run_test_params_arr[#{idx}].name = \"#{test[:test]}\";")
output.puts(" run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
idx += 1
else
test[:args].each.with_index(1) do |args, idx|
wrapper = "runner_args#{idx}_#{test[:test]}"
test[:args].each.with_index(1) do |args, arg_idx|
wrapper = "runner_args#{arg_idx}_#{test[:test]}"
testname = "#{test[:test]}(#{args})".dump
output.puts(" run_test(#{wrapper}, #{testname}, #{test[:line_number]});")
output.puts(" run_test_params_arr[#{idx}].func = #{wrapper};")
output.puts(" run_test_params_arr[#{idx}].name = #{testname};")
output.puts(" run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
idx += 1
end
end
end
output.puts
if @options[:shuffle_tests]
output.puts(' shuffleTests(run_test_params_arr, number_of_tests);')
output.puts
end
output.puts(' for (int i = 0; i < number_of_tests; i++)')
output.puts(' {')
output.puts(' run_test(run_test_params_arr[i].func, run_test_params_arr[i].name, run_test_params_arr[i].line_num);')
output.puts(' }')
output.puts
output.puts(' CMock_Guts_MemFreeFinal();') unless used_mocks.empty?
if @options[:has_suite_teardown]
if @options[:omit_begin_end]
Expand Down Expand Up @@ -534,7 +616,9 @@ def create_h_file(output, filename, tests, testfile_includes, used_mocks)
' --suite_teardown="" - code to execute for teardown of entire suite',
' --use_param_tests=1 - enable parameterized tests (disabled by default)',
' --omit_begin_end=1 - omit calls to UnityBegin and UnityEnd (disabled by default)',
' --header_file="" - path/name of test header file to generate too'].join("\n")
' --header_file="" - path/name of test header file to generate too',
' --shuffle_tests=1 - enable shuffling of the test execution order (disabled by default)',
' --rng_seed=1 - seed value for randomization of test execution order'].join("\n")
exit 1
end

Expand Down
12 changes: 12 additions & 0 deletions docs/UnityHelperScriptsGuide.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,18 @@ Provide any `cdecl` for the `main()` test function. Is empty by default.
If `true`, the `UnityBegin` and `UnityEnd` function will not be called for
Unity test state setup and cleanup.

##### `:shuffle_tests`

If `true`, the test execution order will be shuffled. Is `false` by default.

##### `:rng_seed`

If set to some positive integer value, said value will be used as the seed value passed
to the `srand` function. Otherwise, if not set to any value, `time(NULL)` will be used
as the seed value.

Only applicable if `:shuffle_tests` is set to `true`.

#### Parameterized tests provided macros

Unity provides support for few param tests generators, that can be combined
Expand Down

0 comments on commit 22ed956

Please sign in to comment.