Skip to content

Commit

Permalink
Add support for randomizing test execution order
Browse files Browse the repository at this point in the history 
This commit reintroduces the option to shuffle the test execution order
into the test runner. This has been tested with the temp_sensor example
project in Ceedling.

Signed-off-by: James Raphael Tiovalen <[email protected]>
  • Loading branch information
@jamestiotio
jamestiotio committed Jan 30, 2024
1 parent 64939db commit fbd6120
Showing 1 changed file with 89 additions and 8 deletions.
97 changes: 89 additions & 8 deletions auto/generate_test_runner.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ def self.default_options
use_param_tests: false,
use_system_files: true,
include_extensions: '(?:hpp|hh|H|h)',
source_extensions: '(?:cpp|cc|ino|C|c)'
source_extensions: '(?:cpp|cc|ino|C|c)',
shuffle_tests: false,
rng_seed: 0
}
end

Expand Down Expand Up @@ -90,6 +92,7 @@ def run(input_file, output_file, options = nil)
def generate(input_file, output_file, tests, used_mocks, testfile_includes)
File.open(output_file, 'w') do |output|
create_header(output, used_mocks, testfile_includes)
create_run_test_params_struct(output)
create_externs(output, tests, used_mocks)
create_mock_management(output, used_mocks)
create_setup(output)
Expand All @@ -99,6 +102,7 @@ def generate(input_file, output_file, tests, used_mocks, testfile_includes)
create_reset(output)
create_run_test(output) unless tests.empty?
create_args_wrappers(output, tests)
create_shuffle_tests(output) if @options[:shuffle_tests]
create_main(output, input_file, tests, used_mocks)
end

Expand Down Expand Up @@ -231,16 +235,40 @@ def find_setup_and_teardown(source)
@options[:has_suite_teardown] ||= (source =~ /int\s+suiteTearDown\s*\(int\s+([a-zA-Z0-9_])+\s*\)/)
end

def count_tests(tests)
if (@options[:use_param_tests])
idx = 0
tests.each do |test|
if ((test[:args].nil?) or (test[:args].empty?))
idx += 1
else
test[:args].each do |args|
idx += 1
end
end
end
return idx
else
return tests.size
end
end

def create_header(output, mocks, testfile_includes = [])
output.puts('/* AUTOGENERATED FILE. DO NOT EDIT. */')
output.puts("\n/*=======Automagically Detected Files To Include=====*/")
output.puts('extern "C" {') if @options[:externcincludes]
if @options[:shuffle_tests]
output.puts('#include <stdlib.h>')
if @options[:rng_seed] == 0
output.puts('#include <time.h>')
end
end
output.puts("#include \"#{@options[:framework]}.h\"")
output.puts('#include "cmock.h"') unless mocks.empty?
output.puts('}') if @options[:externcincludes]
if @options[:defines] && !@options[:defines].empty?
output.puts("/* injected defines for unity settings, etc */")
@options[:defines].each do |d|
@options[:defines].each do |d|
def_only = d.match(/(\w+).*/)[1]
output.puts("#ifndef #{def_only}\n#define #{d}\n#endif /* #{def_only} */")
end
Expand Down Expand Up @@ -270,6 +298,16 @@ def create_header(output, mocks, testfile_includes = [])
output.puts('char* GlobalOrderError;')
end

def create_run_test_params_struct(output)
output.puts("\n/*=======Structure Used By Test Runner=====*/")
output.puts('struct UnityRunTestParameters')
output.puts('{')
output.puts(' UnityTestFunction func;')
output.puts(' const char* name;')
output.puts(' UNITY_LINE_TYPE line_num;')
output.puts('};')
end

def create_externs(output, tests, _mocks)
output.puts("\n/*=======External Functions This Runner Calls=====*/")
output.puts("extern void #{@options[:setup_name]}(void);")
Expand Down Expand Up @@ -392,6 +430,22 @@ def create_args_wrappers(output, tests)
end
end

def create_shuffle_tests(output)
output.puts("\n/*=======Shuffle Test Order=====*/")
output.puts('static void shuffleTests(struct UnityRunTestParameters run_test_params_arr[], int num_of_tests)')
output.puts('{')

# Use Fisher-Yates shuffle algorithm
output.puts(' for (int i = 0; i < num_of_tests - 1; i++)')
output.puts(' {')
output.puts(' int j = (rand() % (num_of_tests - i)) + i;')
output.puts(' struct UnityRunTestParameters temp = run_test_params_arr[i];')
output.puts(' run_test_params_arr[i] = run_test_params_arr[j];')
output.puts(' run_test_params_arr[j] = temp;')
output.puts(' }')
output.puts('}')
end

def create_main(output, filename, tests, used_mocks)
output.puts("\n/*=======MAIN=====*/")
main_name = @options[:main_name].to_sym == :auto ? "main_#{filename.gsub('.c', '')}" : (@options[:main_name]).to_s
Expand Down Expand Up @@ -437,18 +491,43 @@ def create_main(output, filename, tests, used_mocks)
else
output.puts(" UnityBegin(\"#{filename.gsub(/\\/, '\\\\\\')}\");")
end
tests.each do |test|
if @options[:shuffle_tests]
output.puts
if @options[:rng_seed] == 0
output.puts(' srand(time(NULL));')
else
output.puts(" srand(#{@options[:rng_seed]});")
end
end
output.puts
output.puts(" int number_of_tests = #{count_tests(tests)};")
output.puts(' struct UnityRunTestParameters run_test_params_arr[number_of_tests];')
output.puts
tests.each.with_index() do |test, idx|
if (!@options[:use_param_tests]) || test[:args].nil? || test[:args].empty?
output.puts(" run_test(#{test[:test]}, \"#{test[:test]}\", #{test[:line_number]});")
output.puts(" run_test_params_arr[#{idx}].func = #{test[:test]};")
output.puts(" run_test_params_arr[#{idx}].name = \"#{test[:test]}\";")
output.puts(" run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
else
test[:args].each.with_index(1) do |args, idx|
wrapper = "runner_args#{idx}_#{test[:test]}"
test[:args].each.with_index(1) do |args, arg_idx|
wrapper = "runner_args#{arg_idx}_#{test[:test]}"
testname = "#{test[:test]}(#{args})".dump
output.puts(" run_test(#{wrapper}, #{testname}, #{test[:line_number]});")
output.puts(" run_test_params_arr[#{idx}].func = #{wrapper};")
output.puts(" run_test_params_arr[#{idx}].name = #{testname};")
output.puts(" run_test_params_arr[#{idx}].line_num = #{test[:line_number]};")
end
end
end
output.puts
if @options[:shuffle_tests]
output.puts(' shuffleTests(run_test_params_arr, number_of_tests);')
output.puts
end
output.puts(' for (int i = 0; i < number_of_tests; i++)')
output.puts(' {')
output.puts(' run_test(run_test_params_arr[i].func, run_test_params_arr[i].name, run_test_params_arr[i].line_num);')
output.puts(' }')
output.puts
output.puts(' CMock_Guts_MemFreeFinal();') unless used_mocks.empty?
if @options[:has_suite_teardown]
if @options[:omit_begin_end]
Expand Down Expand Up @@ -534,7 +613,9 @@ def create_h_file(output, filename, tests, testfile_includes, used_mocks)
' --suite_teardown="" - code to execute for teardown of entire suite',
' --use_param_tests=1 - enable parameterized tests (disabled by default)',
' --omit_begin_end=1 - omit calls to UnityBegin and UnityEnd (disabled by default)',
' --header_file="" - path/name of test header file to generate too'].join("\n")
' --header_file="" - path/name of test header file to generate too',
' --shuffle_tests=1 - enable shuffling of the test execution order (disabled by default)',
' --rng_seed=1 - seed value for randomization of test execution order'].join("\n")
exit 1
end

Expand Down

0 comments on commit fbd6120

Please sign in to comment.