Think about filtering out matching distributions

benchmarks/scripts/analyze.py

@@ -8,6 +8,7 @@
 import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
+from scipy.stats import mannwhitneyu
 
 
 def get_bench_columns():
@@ -146,6 +147,23 @@ def extract_scores(df):
     return result.sort_values(by=['score'], ascending=False)
 
 
+def distributions_are_different(alpha, row):
+    ref_samples = row['base_samples']
+    cmp_samples = row['samples']
+
+    # H0: the distributions are not different
+    # H1: the distribution are different
+    _, p = mannwhitneyu(ref_samples, cmp_samples)
+
+    # Reject H0
+    return p < alpha
+
+
+def remove_matching_distributions(alpha, df):
+    closure = functools.partial(distributions_are_different, alpha)
+    return df[df.apply(closure, axis=1)]
+
+
 def iterate_case_dfs(args, callable):
     storage = cub.bench.Storage()
     pattern = re.compile(args.R)
@@ -172,13 +190,14 @@ def iterate_case_dfs(args, callable):
                     callable(algname, point_str, case_df)
 
 
-def case_top(N, algname, ct_point_name, case_df):
+def case_top(alpha, N, algname, ct_point_name, case_df):
     print("{}[{}]:".format(algname, ct_point_name))
+    case_df = extract_complete_variants(remove_matching_distributions(alpha, case_df))
     print(extract_scores(case_df).head(N))
 
 
 def top(args):
-    iterate_case_dfs(args, functools.partial(case_top, args.top))
+    iterate_case_dfs(args, functools.partial(case_top, args.alpha, args.top))
 
 
 def case_coverage(algname, ct_point_name, case_df):
@@ -280,6 +299,8 @@ def parse_arguments():
         '--coverage', action=argparse.BooleanOptionalAction, help="Show variant space coverage.")
     parser.add_argument(
         '--top', default=7, type=int, action='store', nargs='?', help="Show top N variants with highest score.")
+    parser.add_argument(
+        '--alpha', default=0.5, type=float)
     parser.add_argument(
         '--variants', type=str, help="Show matching variants data.")
     return parser.parse_args()