Refactor code structure for improved readability and maintainability

Author: spyrchat

README.md

@@ -21,11 +21,11 @@ benchmark:
     top_k: 20
 embedding:
   dense:
-    api_key_env: GOOGLE_API_KEY
+    api_key_env: VOYAGE_API_KEY
     batch_size: 32
-    dimensions: 768
-    model: models/embedding-001
-    provider: google
+    dimensions: 1024
+    model: voyage-3.5-lite
+    provider: voyage
     vector_name: dense
   sparse:
     model: Qdrant/bm25

config.yml

@@ -3,6 +3,8 @@
 from embedding.sparse_embedder import SparseEmbedder
 from langchain_qdrant import FastEmbedSparse
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain_voyageai import VoyageAIEmbeddings
+
 import os
 
 
@@ -35,9 +37,39 @@ def get_embedder(cfg: dict):
 
     elif provider == "google":
         model_name = cfg.get("model", "models/embedding-001")
-        return GoogleGenerativeAIEmbeddings(
+        dimensions = cfg.get("dimensions") or cfg.get("output_dimensionality")
+        api_key = os.getenv("GOOGLE_API_KEY")
+
+        if not api_key:
+            raise ValueError(
+                "GOOGLE_API_KEY environment variable is required for Google embeddings")
+
+        # Initialize with or without dimensions parameter
+        if dimensions:
+            return GoogleGenerativeAIEmbeddings(
+                model=model_name,
+                google_api_key=api_key,
+                output_dimensionality=dimensions
+            )
+        else:
+            return GoogleGenerativeAIEmbeddings(
+                model=model_name,
+                google_api_key=api_key
+            )
+
+    elif provider == "voyage":
+        model_name = cfg.get("model", "voyage-3.5-lite")
+        api_key = os.getenv("VOYAGE_API_KEY")
+
+        if not api_key:
+            raise ValueError(
+                "VOYAGE_API_KEY environment variable is required for Voyage embeddings")
+
+        # VoyageAI embeddings use native dimensions (1024 for voyage-3.5)
+        # Dimension reduction can be handled via truncation if needed
+        return VoyageAIEmbeddings(
             model=model_name,
-            google_api_key=os.getenv("GOOGLE_API_KEY")
+            voyage_api_key=api_key
         )
 
     elif provider == "sparse":
@@ -48,5 +80,5 @@ def get_embedder(cfg: dict):
 
     else:
         raise ValueError(
-            f"Unsupported embedder provider: '{provider}'. Supported: hf, titan, fastembed, sparse, google"
+            f"Unsupported embedder provider: '{provider}'. Supported: hf, titan, fastembed, sparse, google, voyage"
         )

embedding/factory.py

@@ -6,24 +6,21 @@ dataset:
   description: "SOSum: Extractive summaries of Stack Overflow posts (506 questions, 2278 posts)"
 
 # Embedding strategy
-embedding_strategy: hybrid
-
 embedding:
+  strategy: "hybrid"
   dense:
-    provider: hf
-    model_name: sentence-transformers/all-MiniLM-L6-v2
+    provider: "google"
+    model: "models/embedding-001"
     batch_size: 32
-    device: cuda
-    vector_name: dense
   sparse:
-    provider: fastembed
-    model_name: Qdrant/bm25
-    vector_name: sparse
+    provider: "sparse"
+    model: "Qdrant/bm25"
+    batch_size: 32
 
 # Chunking configuration
 chunking:
-  strategy: code_aware  # Best for code-heavy content
-  chunk_size: 800       # Larger chunks for code context
+  strategy: "code_aware"  # Best for code-heavy content
+  chunk_size: 800         # Larger chunks for code context
   chunk_overlap: 100
   preserve_functions: true
   preserve_code_blocks: true
@@ -39,14 +36,14 @@ validation:
 
 # Retriever configuration
 retriever:
-  type: qdrant
+  type: "qdrant"
   top_k: 15
 
 # Qdrant settings
 qdrant:
-  collection: sosum_stackoverflow_v1
-  dense_vector_name: dense
-  sparse_vector_name: sparse
+  collection: "sosum_stackoverflow_v1"
+  dense_vector_name: "dense"
+  sparse_vector_name: "sparse"
 
 # Upload settings
 upload:

pipelines/configs/datasets/stackoverflow.yml

@@ -1,44 +1,43 @@
-# Configuration for Hybrid Dense + Sparse Embeddings
-
 dataset:
   name: "stackoverflow_sosum"
   version: "v1.0.0"
-  adapter: "stackoverflow"  # REQUIRED: This was missing!
+  adapter: "stackoverflow"
 
 chunking:
-  strategy: "recursive"  # FIXED: Strategy name
+  strategy: "recursive"
   chunk_size: 512
   chunk_overlap: 50
-  separators: ["\n\n", "\n", " ", ""]  # REQUIRED for recursive chunking
+  separators: ["\n\n", "\n", " ", ""]
 
 embedding:
-  strategy: "hybrid"  # FIXED: Moved from top level
+  strategy: "hybrid"
   dense:
     provider: "google"
-    model: "models/embedding-001"  # FIXED: changed from model_name to model
+    model: "models/embedding-001"
+    dimensions: 1536        # Available: 128, 256, 512, 768, 1536, 3072 (default: 3072)
     batch_size: 32
   sparse:
     provider: "sparse"
-    model: "Qdrant/bm25"  # FIXED: changed from model_name to model
+    model: "Qdrant/bm25"
     batch_size: 32
 
 qdrant:
   collection: "sosum_stackoverflow_hybrid_v1"
   dense_vector_name: "dense"
   sparse_vector_name: "sparse"
-  distance_metric: "cosine"  # REQUIRED: Added missing field
+  distance_metric: "cosine"
 
 upload:
   batch_size: 50
   wait: true
-  versioning: true  # ADDED: For proper versioning
+  versioning: true
 
 validation:
-  enabled: true  # REQUIRED: Added missing field
-  max_text_length: 10000  # FIXED: changed from max_char_length
-  min_text_length: 10     # FIXED: changed from min_char_length
+  enabled: true
+  max_text_length: 10000
+  min_text_length: 10
 
 smoke_tests:
   enabled: true
-  sample_size: 5  # REQUIRED: Added missing field
+  sample_size: 5
   min_success_rate: 0.7

pipelines/configs/datasets/stackoverflow_hybrid.yml

@@ -0,0 +1,36 @@
+embedding:
+  strategy: "hybrid"
+  dense:
+    provider: "voyage"
+    model: "voyage-3.5-lite"
+    batch_size: 32
+    dimensions: 1024
+  sparse:
+    provider: "sparse"
+    model: "Qdrant/bm25"
+    batch_size: 8
+
+qdrant:
+  collection: "stackoverflow_voyage_lite"
+  host: "localhost"
+  port: 6333
+  timeout: 300
+  distance: "Cosine"
+  
+chunking:
+  method: "recursive"
+  chunk_size: 500
+  chunk_overlap: 100
+  separators: ["\n\n", "\n", " ", ""]
+
+processing:
+  validate_documents: true
+  enable_duplicate_detection: true
+  similarity_threshold: 0.95
+  batch_size: 100
+  max_retries: 3
+  retry_delay: 1.0
+
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"

pipelines/configs/datasets/stackoverflow_voyage_lite.yml

@@ -0,0 +1,68 @@
+# SOSum Stack Overflow Dataset Configuration with Voyage AI Premium Embeddings
+# Dataset: https://github.com/BonanKou/SOSum-A-Dataset-of-Extractive-Summaries-of-Stack-Overflow-Posts-and-labeling-tools
+dataset:
+  name: "stackoverflow_sosum"
+  version: "v1.0.0"
+  adapter: "stackoverflow"
+
+chunking:
+  strategy: "recursive"
+  chunk_size: 512
+  chunk_overlap: 50
+  separators: ["\n\n", "\n", " ", ""]
+
+embedding:
+  strategy: "hybrid"
+  dense:
+    provider: "voyage"
+    model: "voyage-3.5"           # Premium option: $0.06/1M tokens (better quality)
+    dimensions: 1024              # Full dimension for maximum quality
+    batch_size: 32
+  sparse:
+    provider: "sparse"
+    model: "Qdrant/bm25"
+    batch_size: 32
+
+qdrant:
+  collection: "sosum_stackoverflow_voyage_premium_v1"
+  dense_vector_name: "dense"
+  sparse_vector_name: "sparse"
+
+upload:
+  batch_size: 50
+  wait: true
+  versioning: true
+
+validation:
+  min_char_length: 30
+  max_char_length: 50000
+  remove_duplicates: true
+  clean_html: true
+  preserve_code_blocks: true
+  allowed_languages: ["en"]
+
+# Evaluation settings
+evaluation:
+  k_values: [1, 3, 5, 10, 15]
+  similarity_threshold: 0.7
+
+# Smoke tests
+smoke_tests:
+  min_success_rate: 0.8
+  golden_queries:
+    - query: "Python list comprehension example"
+      min_recall: 0.1
+    - query: "JavaScript async function"
+      min_recall: 0.1
+    - query: "How to solve error in code"
+      min_recall: 0.1
+    - query: "Best practice programming"
+      min_recall: 0.1
+
+# Output configuration
+output_dir: "output/sosum_stackoverflow_voyage_premium"
+
+# Embedding cache
+embedding_cache:
+  enabled: true
+  dir: "cache/embeddings/sosum_stackoverflow_voyage_premium"