From 07c3f10da46dbbcc6f7bb8d1433efad6577d4d69 Mon Sep 17 00:00:00 2001 From: Iavor Diatchki Date: Wed, 11 Dec 2024 16:20:27 -0800 Subject: [PATCH] Scripts to run mutation and unit tests. --- preprocessor/README.md | 3 +- preprocessor/config | 11 +++++ preprocessor/example.c | 39 ++++++++++++++++ preprocessor/preproc_tut.ml | 92 ++++++++++++++++++++++--------------- preprocessor/run-all | 26 +++++++++++ preprocessor/run-mutant | 30 ++++++++++++ preprocessor/run-unit-test | 30 ++++++++++++ 7 files changed, 191 insertions(+), 40 deletions(-) create mode 100644 preprocessor/config create mode 100644 preprocessor/example.c create mode 100755 preprocessor/run-all create mode 100755 preprocessor/run-mutant create mode 100755 preprocessor/run-unit-test diff --git a/preprocessor/README.md b/preprocessor/README.md index a4d00d0..256b79a 100644 --- a/preprocessor/README.md +++ b/preprocessor/README.md @@ -28,7 +28,7 @@ Notation for Mutation Testing The pre-processor is line based. For mutation testing we use a CPP-like if-block, as illustrated by the following example: ``` -#if !MUTATION +#if !MUTATION(function_containing_the_mutant) Normal code path @@ -41,7 +41,6 @@ Some other variant #endif ``` - If we run the pre-processer to eliminate mutation testing the result would be only: diff --git a/preprocessor/config b/preprocessor/config new file mode 100644 index 0000000..a1e4548 --- /dev/null +++ b/preprocessor/config @@ -0,0 +1,11 @@ + +PREPROC="$SCRIPT_DIR/preproc_tut" + +CC="gcc" +# -P: do not emit line pragmas +# -CC: do not discard comments +CPP="$CC -E -P -CC" + +CN="cn" +CN_INSTALL="$OPAM_SWITCH_PREFIX/lib/cn/runtime" + diff --git a/preprocessor/example.c b/preprocessor/example.c new file mode 100644 index 0000000..f926018 --- /dev/null +++ b/preprocessor/example.c @@ -0,0 +1,39 @@ +// This file show how we envision using the preprocessor. +#define S 11111111 + +int puts(const char*); + +int some_other_fun(int x) +/*@ requires true; ensures return == 1i32; @*/ +{ + return 0; +} + +int inc(int x) +/*@ requires 0i32 <= x && x < 10i32; ensures return < 11i32; @*/ +{ +#if !MUTATION(inc) + return x + 1; +#elif X_PLUS_2 + return x + 2; +#elif X_PLUS_3 + return x + 3; +#endif +} + + +#if CN_TEST_A +int main(int argc, char* argv[]) { + return inc(S); +} +#endif + +#if CN_TEST_B +int main(int argc, char* argv[]) { + return inc(5); +} +#endif + + + + diff --git a/preprocessor/preproc_tut.ml b/preprocessor/preproc_tut.ml index 006cc61..dfd168a 100644 --- a/preprocessor/preproc_tut.ml +++ b/preprocessor/preproc_tut.ml @@ -7,11 +7,22 @@ let drop_prefix prefix str = Some (String.trim (String.sub str n (l - n))) else None +let rec drop_prefixes prefixes str = + match prefixes with + | [] -> Some str + | p :: ps -> + match drop_prefix p str with + | Some rest -> drop_prefixes ps rest + | None -> None + (* Should we start a new mutant block *) let start_mutant_block line = - match drop_prefix "#if" line with - | Some "!MUTATION" -> true - | _ -> false + match drop_prefixes ["#if"; "!"; "MUTATION"; "("] line with + | Some res when String.ends_with ~suffix:")" res -> + let fu = String.trim (String.sub res 0 (String.length res - 1)) in + Some fu + | _ -> None + (* Does this line start a mutant *) let start_mutant = drop_prefix "#elif" @@ -42,8 +53,8 @@ type mode = (* The current state of the processor *) type state = | TopLevel -| InMutantOrig of int (* start line, for error reprorting *) -| InMutant of (int * string) (* start line; mutant name *) +| InMutantOrig of (int * string) (* start line, function name *) +| InMutant of (int * string * string) (* start line; function name; mutant name *) | InUnitTest of (int * string) (* start line; test name *) @@ -55,54 +66,59 @@ let rec process_input mode start_line state = let mk_error no msg = failwith (Printf.sprintf "%d: %s" no msg) in match state with | TopLevel -> () - | InMutantOrig n -> mk_error n "Untermianted mutant block" - | InMutant (n,_) -> mk_error n "Unterminated mutant block" + | InMutantOrig (n,_)-> mk_error n "Untermianted mutant block" + | InMutant (n,_,_) -> mk_error n "Unterminated mutant block" | InUnitTest (n,_) -> mk_error n "Unterminated unit test" end | Some line -> let new_state = match state with - (* start a mutation test *) - | TopLevel when start_mutant_block line -> - begin match mode with - | MutationTesting -> print_endline "//! //" - | _ -> () - end; - InMutantOrig start_line (* next state *) - + | TopLevel -> - begin match start_unit_test line with - - (* start a unit test *) - | Some name -> - begin match mode with - | CollectUnitTest -> print_endline name - | _ -> () - end; - InUnitTest (start_line, name) (* next state *) + begin match start_mutant_block line with - (* ordinary top level line *) - | None -> - begin match mode with - | CollectUnitTest - | CollectMutants -> () - | _ -> print_endline line - end; - state (* next state *) + (* start a mutation test *) + | Some fu -> + begin match mode with + | MutationTesting -> print_endline "//! //" + | _ -> () + end; + InMutantOrig (start_line,fu) (* next state *) + + | None -> + begin match start_unit_test line with + + (* start a unit test *) + | Some name -> + begin match mode with + | CollectUnitTest -> print_endline name + | _ -> () + end; + InUnitTest (start_line, name) (* next state *) + + (* ordinary top level line *) + | None -> + begin match mode with + | CollectUnitTest + | CollectMutants -> () + | _ -> print_endline line + end; + state (* next state *) + end end - | InMutantOrig ln -> + | InMutantOrig (ln,fu) -> begin match start_mutant line with (* Start a mutant *) | Some name -> begin match mode with | MutationTesting -> Printf.printf "//!! %s // //!\n" name - | CollectMutants -> print_endline name + | CollectMutants -> Printf.printf "%s/%s\n" fu name | _ -> () end; - InMutant (ln,name) (* next state *) + InMutant (ln,fu,name) (* next state *) (* Original part of a mutation block *) | None -> @@ -123,17 +139,17 @@ let rec process_input mode start_line state = end; TopLevel (* next state *) - | InMutant (ln,name) -> + | InMutant (ln,fu,name) -> begin match start_mutant line with (* Next mutatant *) | Some new_name -> begin match mode with | MutationTesting -> Printf.printf "// //!! %s // //!\n" new_name - | CollectMutants -> print_endline new_name + | CollectMutants -> Printf.printf "%s/%s\n" fu new_name | _ -> () end; - InMutant (ln,new_name) (* next state *) + InMutant (ln,fu,new_name) (* next state *) (* Line in a mutant *) | None -> diff --git a/preprocessor/run-all b/preprocessor/run-all new file mode 100755 index 0000000..6294273 --- /dev/null +++ b/preprocessor/run-all @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo "USAGE: $0 CFILE" + echo " Run all unit tests and mutants in a single C file." + exit 1 +fi + +FILE="$1" + +SCRIPT_DIR="$(dirname $0)" +source "$SCRIPT_DIR/config" + +for UNIT_TEST in $($PREPROC --list-unit < "$FILE"); do + echo $UNIT_TEST + $SCRIPT_DIR/run-unit-test "$FILE" "$UNIT_TEST" +done + +for MUTANT in $($PREPROC --list-mutants < "$FILE"); do + echo $MUTANT + $SCRIPT_DIR/run-mutant "$FILE" "$MUTANT" +done + + + + diff --git a/preprocessor/run-mutant b/preprocessor/run-mutant new file mode 100755 index 0000000..b9d6b21 --- /dev/null +++ b/preprocessor/run-mutant @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "USAGE: $0 CFILE FUN_QUALIFIED_MUTANT_NAME" + echo " Preprocess, and test a single mutant in a single C file." + exit 1 +fi + +SCRIPT_DIR="$(dirname $0)" +source "$SCRIPT_DIR/config" + +CFILE="$1" +FUN=$(dirname "$2") +UNIT_TEST=$(basename "$2") + +OUT_DIR=$(mktemp -d /tmp/cn-run-mutant-XXXXXX) +trap "rm -rf $OUT_DIR" EXIT + +cat "$CFILE" | $PREPROC --show-mutant "$UNIT_TEST" > "$OUT_DIR/$UNIT_TEST.c" + +pushd "$OUT_DIR" +$CPP "$UNIT_TEST.c" > "$UNIT_TEST.i" +mv "$UNIT_TEST.i" "$UNIT_TEST.c" +$CN test "$UNIT_TEST.c" --output-dir=. --only="$FUN" --seed=0 > /dev/null +RESULT=$? +popd +exit $RESULT + + + diff --git a/preprocessor/run-unit-test b/preprocessor/run-unit-test new file mode 100755 index 0000000..6aecce8 --- /dev/null +++ b/preprocessor/run-unit-test @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "USAGE: $0 CFILE UNIT_TEST_NAME" + echo " Preprocess, and build, a single test in a single C file." + exit 1 +fi + +SCRIPT_DIR="$(dirname $0)" +source "$SCRIPT_DIR/config" + +CFILE="$1" +UNIT_TEST="$2" + +OUT_DIR=$(mktemp -d /tmp/cn-run-unit-test-XXXXXX) +trap "rm -rf $OUT_DIR" EXIT + +cat "$CFILE" | $PREPROC --show-unit "$UNIT_TEST" > "$OUT_DIR/$UNIT_TEST.c" + +pushd "$OUT_DIR" +$CPP "$UNIT_TEST.c" > "$UNIT_TEST.i" +mv "$UNIT_TEST.i" "$UNIT_TEST.c" +$CN instrument --fail-fast "$UNIT_TEST.c" +$CC -I "$CN_INSTALL/include" -L "$CN_INSTALL" \ + "$UNIT_TEST-exec.c" -lcn -o "$UNIT_TEST" +"./$UNIT_TEST" +popd + + +