automatically add test dataset when doing benches

src/axolotl/utils/config/models/input/v0_4_1/__init__.py

@@ -988,6 +988,20 @@ def check_evals(cls, data):
             )
         return data
 
+    @model_validator(mode="before")
+    @classmethod
+    def check_test_datasets_bench(cls, data):
+        if (
+            data.get("do_bench_eval")
+            and not data.get("test_datasets")
+            and not data.get("val_set_size")
+        ):
+            LOG.warning(
+                "`do_bench_eval` needs a test dataset to run evals, adding an empty test_dataset."
+            )
+            data["test_datasets"] = [{"path": "axolotl-ai-co/empty-test-ds"}]
+        return data
+
     @model_validator(mode="before")
     @classmethod
     def check_eval_packing(cls, data):