Call out limitations of Eleuther generation capabilities (pytorch#1251)

recipes/eleuther_eval.py

@@ -130,6 +130,12 @@ def _model_generate(
     ) -> torch.Tensor:
         curr_batch_size = context.size(0)
 
+        if curr_batch_size > 1:
+            raise ValueError(
+                f"Got a batch size of '{curr_batch_size}'. Batch size > 1 is not supported for "
+                "generation. See https://github.com/pytorch/torchtune/issues/1250 for more info."
+            )
+
         # Setup caches for a given batch size
         # Technically this is not necessary, but it's a good way to ensure that
         # the caches won't error on a different batch size. In addition, caches