fix hiyouga#2397

xin-li-67 · Feb 3, 2024 · 51df865 · 51df865
1 parent 6545c02
commit 51df865
Showing 1 changed file with 0 additions and 3 deletions.
diff --git a/src/llmtuner/data/loader.py b/src/llmtuner/data/loader.py
@@ -155,9 +155,6 @@ def get_dataset(
                 dataset = dataset.to_iterable_dataset()
             return dataset
 
-        if data_args.streaming:
-            raise ValueError("Turn off dataset streaming to save cache files.")
-
     with training_args.main_process_first(desc="load dataset"):
         all_datasets = []
         for dataset_attr in get_dataset_list(data_args):  # TODO: add split