Scripts to run mutation and unit tests.

preprocessor/README.md

@@ -28,7 +28,7 @@ Notation for Mutation Testing
 The pre-processor is line based.  For mutation testing we use a CPP-like
 if-block, as illustrated by the following example:
 ```
-#if !MUTATION
+#if !MUTATION(function_containing_the_mutant)
 Normal
 code
 path
@@ -41,7 +41,6 @@ Some other variant
 #endif
 ```
 
-
 If we run the pre-processer to eliminate mutation testing the result would
 be only:
 

preprocessor/config

@@ -0,0 +1,11 @@
+
+PREPROC="$SCRIPT_DIR/preproc_tut"
+
+CC="gcc"
+# -P: do not emit line pragmas
+# -CC: do not discard comments
+CPP="$CC -E -P -CC"
+
+CN="cn"
+CN_INSTALL="$OPAM_SWITCH_PREFIX/lib/cn/runtime"
+

preprocessor/example.c

@@ -0,0 +1,39 @@
+// This file show how we envision using the preprocessor.
+#define S 11111111
+
+int puts(const char*);
+
+int some_other_fun(int x)
+/*@ requires true; ensures return == 1i32; @*/
+{
+  return 0;
+}
+
+int inc(int x)
+/*@ requires 0i32 <= x && x < 10i32; ensures  return < 11i32; @*/
+{
+#if !MUTATION(inc)
+  return x + 1;
+#elif X_PLUS_2
+  return x + 2;
+#elif X_PLUS_3
+  return x + 3;
+#endif
+}
+
+
+#if CN_TEST_A
+int main(int argc, char* argv[]) {
+  return inc(S);
+}
+#endif
+
+#if CN_TEST_B
+int main(int argc, char* argv[]) {
+  return inc(5);
+}
+#endif
+
+
+
+

preprocessor/preproc_tut.ml

@@ -7,11 +7,22 @@ let drop_prefix prefix str =
       Some (String.trim (String.sub str n (l - n)))
     else None
 
+let rec drop_prefixes prefixes str =
+  match prefixes with
+  | [] -> Some str
+  | p :: ps ->
+    match drop_prefix p str with
+    | Some rest -> drop_prefixes ps rest
+    | None -> None
+
 (* Should we start a new mutant block *)
 let start_mutant_block line =
-  match drop_prefix "#if" line with
-  | Some "!MUTATION" -> true 
-  | _                -> false
+  match drop_prefixes ["#if"; "!"; "MUTATION"; "("] line with
+  | Some res when String.ends_with ~suffix:")" res ->
+    let fu = String.trim (String.sub res 0 (String.length res - 1)) in
+    Some fu
+  | _ -> None
+
 
 (* Does this line start a mutant *)
 let start_mutant = drop_prefix "#elif"
@@ -42,8 +53,8 @@ type mode =
 (* The current state of the processor *)
 type state = 
 | TopLevel
-| InMutantOrig of int            (* start line, for error reprorting *)
-| InMutant     of (int * string) (* start line; mutant name *)
+| InMutantOrig of (int * string) (* start line, function name *)
+| InMutant     of (int * string * string) (* start line; function name; mutant name *)
 | InUnitTest   of (int * string) (* start line; test name *)
 
 
@@ -55,54 +66,59 @@ let rec process_input mode start_line state =
     let mk_error no msg = failwith (Printf.sprintf "%d: %s" no msg) in
     match state with
     | TopLevel          -> ()
-    | InMutantOrig n    -> mk_error n "Untermianted mutant block"
-    | InMutant (n,_)    -> mk_error n "Unterminated mutant block"
+    | InMutantOrig (n,_)-> mk_error n "Untermianted mutant block"
+    | InMutant (n,_,_)  -> mk_error n "Unterminated mutant block"
     | InUnitTest (n,_)  -> mk_error n "Unterminated unit test"
     end
   | Some line ->
     let new_state =
       match state with
 
-      (* start a mutation test *)
-      | TopLevel when start_mutant_block line ->
-        begin match mode with
-        | MutationTesting -> print_endline "//! //"
-        | _               -> ()
-        end;
-        InMutantOrig start_line (* next state *)
-
+
       | TopLevel ->
-        begin match start_unit_test line with
-
-          (* start a unit test *)
-          | Some name ->
-            begin match mode with
-            | CollectUnitTest -> print_endline name
-            | _               -> ()
-            end;
-            InUnitTest (start_line, name) (* next state *)
+        begin match start_mutant_block line with
 
-          (* ordinary top level line *)
-          | None ->
-            begin match mode with
-            | CollectUnitTest
-            | CollectMutants  -> ()
-            | _               -> print_endline line
-            end;
-            state (* next state *)
+        (* start a mutation test *)
+        | Some fu ->
+          begin match mode with
+          | MutationTesting -> print_endline "//! //"
+          | _               -> ()
+          end;
+          InMutantOrig (start_line,fu) (* next state *)
+
+        | None ->
+           begin match start_unit_test line with
+
+             (* start a unit test *)
+             | Some name ->
+               begin match mode with
+               | CollectUnitTest -> print_endline name
+               | _               -> ()
+               end;
+               InUnitTest (start_line, name) (* next state *)
+
+             (* ordinary top level line *)
+             | None ->
+               begin match mode with
+               | CollectUnitTest
+               | CollectMutants  -> ()
+               | _               -> print_endline line
+               end;
+               state (* next state *)
+           end
         end
 
-      | InMutantOrig ln ->
+      | InMutantOrig (ln,fu) ->
         begin match start_mutant line with
 
         (* Start a mutant *)
         | Some name ->
           begin match mode with
           | MutationTesting -> Printf.printf "//!! %s // //!\n" name
-          | CollectMutants  -> print_endline name
+          | CollectMutants  -> Printf.printf "%s/%s\n" fu name
           | _               -> ()
           end;
-          InMutant (ln,name) (* next state *)
+          InMutant (ln,fu,name) (* next state *)
 
         (* Original part of a mutation block *) 
         | None ->
@@ -123,17 +139,17 @@ let rec process_input mode start_line state =
         end;
         TopLevel (* next state *)
 
-      | InMutant (ln,name) ->
+      | InMutant (ln,fu,name) ->
 
         begin match start_mutant line with
         (* Next mutatant *)
         | Some new_name ->
             begin match mode with
             | MutationTesting -> Printf.printf "// //!! %s // //!\n" new_name
-            | CollectMutants  -> print_endline new_name
+            | CollectMutants  -> Printf.printf "%s/%s\n" fu new_name
             | _               -> ()
             end;
-            InMutant (ln,new_name) (* next state *)
+            InMutant (ln,fu,new_name) (* next state *)
 
         (* Line in a mutant *)
         | None ->

preprocessor/run-all

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+  echo "USAGE: $0 CFILE"
+  echo "  Run all unit tests and mutants in a single C file."
+  exit 1
+fi
+
+FILE="$1"
+
+SCRIPT_DIR="$(dirname $0)"
+source "$SCRIPT_DIR/config"
+
+for UNIT_TEST in $($PREPROC --list-unit < "$FILE"); do
+  echo $UNIT_TEST
+  $SCRIPT_DIR/run-unit-test "$FILE" "$UNIT_TEST"
+done
+
+for MUTANT in $($PREPROC --list-mutants < "$FILE"); do
+  echo $MUTANT
+  $SCRIPT_DIR/run-mutant "$FILE" "$MUTANT"
+done
+
+
+
+

preprocessor/run-mutant

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ $# -ne 2 ]; then
+  echo "USAGE: $0 CFILE FUN_QUALIFIED_MUTANT_NAME"
+  echo "  Preprocess, and test a single mutant in a single C file."
+  exit 1
+fi
+
+SCRIPT_DIR="$(dirname $0)"
+source "$SCRIPT_DIR/config"
+
+CFILE="$1"
+FUN=$(dirname "$2")
+UNIT_TEST=$(basename "$2")
+
+OUT_DIR=$(mktemp -d /tmp/cn-run-mutant-XXXXXX)
+trap "rm -rf $OUT_DIR" EXIT
+
+cat "$CFILE" | $PREPROC --show-mutant "$UNIT_TEST" > "$OUT_DIR/$UNIT_TEST.c"
+
+pushd "$OUT_DIR"
+$CPP "$UNIT_TEST.c" > "$UNIT_TEST.i"
+mv "$UNIT_TEST.i" "$UNIT_TEST.c"
+$CN test "$UNIT_TEST.c" --output-dir=. --only="$FUN" --seed=0 > /dev/null
+RESULT=$?
+popd
+exit $RESULT
+
+
+

preprocessor/run-unit-test

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ $# -ne 2 ]; then
+  echo "USAGE: $0 CFILE UNIT_TEST_NAME"
+  echo "  Preprocess, and build, a single test in a single C file."
+  exit 1
+fi
+
+SCRIPT_DIR="$(dirname $0)"
+source "$SCRIPT_DIR/config"
+
+CFILE="$1"
+UNIT_TEST="$2"
+
+OUT_DIR=$(mktemp -d /tmp/cn-run-unit-test-XXXXXX)
+trap "rm -rf $OUT_DIR" EXIT
+
+cat "$CFILE" | $PREPROC --show-unit "$UNIT_TEST" > "$OUT_DIR/$UNIT_TEST.c"
+
+pushd "$OUT_DIR"
+$CPP "$UNIT_TEST.c" > "$UNIT_TEST.i"
+mv "$UNIT_TEST.i" "$UNIT_TEST.c"
+$CN instrument --fail-fast "$UNIT_TEST.c"
+$CC -I "$CN_INSTALL/include" -L "$CN_INSTALL" \
+  "$UNIT_TEST-exec.c" -lcn -o "$UNIT_TEST"
+"./$UNIT_TEST"
+popd
+
+
+