An informative comment.

JuliaGenAI · Jun 2, 2024 · a9b266d · a9b266d
1 parent b73c39a
commit a9b266d
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/modelling/tokenization/doc_tokenization.jl b/src/modelling/tokenization/doc_tokenization.jl
@@ -27,6 +27,7 @@ function tensorize(doc_tokenizer::DocTokenizer, tokenizer::Transformers.TextEnco
     if ismissing(bsize)
         return integer_ids, integer_mask
     else
+        # we sort passages by length to do batch packing for more efficient use of the GPU
         integer_ids, integer_mask, reverse_indices = _sort_by_length(integer_ids, integer_mask, bsize)
         batches = _split_into_batches(integer_ids, integer_mask, bsize)