Merge pull request #18 from codetalker7/query_embeddings

Fixing indexing code to include `QueryTokenizer` in the `Checkpoint`.
JuliaGenAI · Jul 24, 2024 · 76e85ad · 76e85ad
2 parents a8f332d + a4c7f85
commit 76e85ad
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/indexing.jl b/src/indexing.jl
@@ -17,7 +17,7 @@ function index(indexer::Indexer)
     # loading the models
     @info "Loading ColBERT layers from HuggingFace."
     base_colbert = BaseColBERT(checkpoint, config)
-    checkPoint = Checkpoint(base_colbert, DocTokenizer(base_colbert.tokenizer, config), config)
+    checkPoint = Checkpoint(base_colbert, DocTokenizer(base_colbert.tokenizer, config), QueryTokenizer(base_colbert.tokenizer, config), config)
 
     # creating the encoder, saver and indexer
     encoder = CollectionEncoder(config, checkPoint)