Added test filter

ut-amrl · Jul 13, 2023 · 63bd004 · 63bd004
1 parent 009b807
commit 63bd004
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/codebotler_benchmark.py b/codebotler_benchmark.py
@@ -64,7 +64,7 @@ def generate(args):
 def evaluate(args):
   print(f"Evaluating completions from {args.generate_output}...")
   print(f"Benchmark file: {args.benchmark_file}")
-  evaluate_trace(args.generate_output, args.evaluate_output, print_completions=args.print_completions)
+  evaluate_trace(args.generate_output, args.evaluate_output, print_completions=args.print_completions, test_filter=args.test_filter)
 
 
 def main():
@@ -92,6 +92,7 @@ def main():
   parser.add_argument("--top-p", type=float, default=0.95)
   parser.add_argument("--temperature", type=float, default=0.2)
   parser.add_argument("--print-completions", action="store_true", help="Print completions to stdout")
+  parser.add_argument("--test-filter", type=str, help="Regex to filter tests to run.")
 
   # For an automodel
   parser.add_argument("--batch-size", type=int, default=1)