Vector Search based of Similarity Threshold #14278
Unanswered
PradiptaBehera8558
asked this question in
Q&A
Replies: 1 comment 2 replies
-
To perform a vector search based on a similarity score threshold of 0.4 using Qdrant as your vector store, you should use the Here is an example of how you can set it up: from llama_index.core import VectorStoreIndex, StorageContext
from llama_index.vector_stores.qdrant import QdrantVectorStore
from qdrant_client import QdrantClient, AsyncQdrantClient
from llama_index.legacy.indices.vector_store.retrievers.retriever import VectorIndexRetriever
# Create a persistent index to disk
client = QdrantClient(host="localhost", port=6333)
aclient = AsyncQdrantClient(host="localhost", port=6333)
# Create our vector store
vector_store = QdrantVectorStore(
"llama2_paper",
client=client,
aclient=aclient,
enable_hybrid=True,
batch_size=20,
)
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex.from_documents(
documents,
storage_context=storage_context,
)
# Customizing the hybrid fusion function to include a similarity score threshold
from llama_index.legacy.vector_stores.types import VectorStoreQueryResult
def relative_score_fusion_with_threshold(
dense_result: VectorStoreQueryResult,
sparse_result: VectorStoreQueryResult,
alpha: float = 0.5, # passed in from the query engine
top_k: int = 2, # passed in from the query engine i.e. similarity_top_k
threshold: float = 0.4 # similarity score threshold
) -> VectorStoreQueryResult:
"""
Fuse dense and sparse results using relative score fusion with a similarity score threshold.
"""
# Sanity check
assert dense_result.nodes is not None
assert dense_result.similarities is not None
assert sparse_result.nodes is not None
assert sparse_result.similarities is not None
# Deconstruct results
sparse_result_tuples = list(
zip(sparse_result.similarities, sparse_result.nodes)
)
sparse_result_tuples.sort(key=lambda x: x[0], reverse=True)
dense_result_tuples = list(
zip(dense_result.similarities, dense_result.nodes)
)
dense_result_tuples.sort(key=lambda x: x[0], reverse=True)
# Track nodes in both results
all_nodes_dict = {x.node_id: x for x in dense_result.nodes}
for node in sparse_result.nodes:
if node.node_id not in all_nodes_dict:
all_nodes_dict[node.node_id] = node
# Normalize sparse similarities from 0 to 1
sparse_similarities = [x[0] for x in sparse_result_tuples]
max_sparse_sim = max(sparse_similarities)
min_sparse_sim = min(sparse_similarities)
sparse_similarities = [
(x - min_sparse_sim) / (max_sparse_sim - min_sparse_sim)
for x in sparse_similarities
]
sparse_per_node = {
sparse_result_tuples[i][1].node_id: x
for i, x in enumerate(sparse_similarities)
}
# Normalize dense similarities from 0 to 1
dense_similarities = [x[0] for x in dense_result_tuples]
max_dense_sim = max(dense_similarities)
min_dense_sim = min(dense_similarities)
dense_similarities = [
(x - min_dense_sim) / (max_dense_sim - min_dense_sim)
for x in dense_similarities
]
dense_per_node = {
dense_result_tuples[i][1].node_id: x
for i, x in enumerate(dense_similarities)
}
# Fuse the scores
fused_similarities = []
for node_id in all_nodes_dict:
sparse_sim = sparse_per_node.get(node_id, 0)
dense_sim = dense_per_node.get(node_id, 0)
fused_sim = alpha * (sparse_sim + dense_sim)
if fused_sim >= threshold:
fused_similarities.append((fused_sim, all_nodes_dict[node_id]))
fused_similarities.sort(key=lambda x: x[0], reverse=True)
fused_similarities = fused_similarities[:top_k]
# Create final response object
return VectorStoreQueryResult(
nodes=[x[1] for x in fused_similarities],
similarities=[x[0] for x in fused_similarities],
ids=[x[1].node_id for x in fused_similarities],
)
# Use the custom fusion function in the vector store
vector_store = QdrantVectorStore(
"llama2_paper",
client=client,
enable_hybrid=True,
hybrid_fusion_fn=relative_score_fusion_with_threshold,
)
# Set up the retriever
retriever = VectorIndexRetriever(
index=index,
similarity_top_k=2,
vector_store_query_mode="hybrid",
alpha=0.5,
vector_store_kwargs={"hybrid_fusion_fn": relative_score_fusion_with_threshold}
)
# Perform the query
query_bundle = QueryBundle(query_str="your query here")
results = retriever._retrieve(query_bundle) This setup ensures that only nodes with a similarity score above the threshold of 0.4 are considered in the final results [1][2][3]. |
Beta Was this translation helpful? Give feedback.
2 replies
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
-
Hi Team
I would like to perform vector search based on similarity score threshold.
For example, suppose similarity threshold 0.4, all documents needs to be returned those similarity score is beyond 0.4
For this which retriever should I use. Consider my vector store is Qdrant
Beta Was this translation helpful? Give feedback.
All reactions