chore: use "BatchLongest"

Anush008 · Jan 31, 2024 · 770a12e · 770a12e
1 parent 25993ce
commit 770a12e
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/fastembed.go b/fastembed.go
@@ -362,13 +362,13 @@ func loadTokenizer(modelPath string, maxLength int) (*tokenizer.Tokenizer, error
 		Stride:    0,
 	})
 
-	paddingStrategy := tokenizer.NewPaddingStrategy(tokenizer.WithFixed(maxLength))
-
 	paddingParams := tokenizer.PaddingParams{
-		Strategy:  *paddingStrategy,
+		// Strategy defaults to "BatchLongest"
+		Strategy:  *tokenizer.NewPaddingStrategy(),
 		Direction: tokenizer.Right,
 		PadId:     int(config["pad_token_id"].(float64)),
 		PadToken:  tokenizerConfig["pad_token"].(string),
+		PadTypeId: 0,
 	}
 	tknzer.WithPadding(&paddingParams)