fix: do padding ourself

PrimeIntellect-ai · Sep 25, 2024 · a90d07e · a90d07e
1 parent c0eb41a
commit a90d07e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/zeroband/data.py b/src/zeroband/data.py
@@ -69,7 +69,7 @@ def get_dataloader(
         ds = load_dataset("allenai/c4", "en", streaming=True)
 
         def tokenize_function(data):
-            outputs = tokenizer(data["text"], truncation=True, max_length=seq_length, padding="max_length")
+            outputs = tokenizer(data["text"], truncation=True, max_length=seq_length)
             return outputs
 
         tokenized_datasets = ds.map(