config.example.yaml

# ╔══════════════════════════════════════════════════════════════════════════════╗
# ║                                                                              ║
# ║                    KNOWLEDGE RAG — Configuration File                        ║
# ║                                                                              ║
# ╚══════════════════════════════════════════════════════════════════════════════╝
#
# This file controls how Knowledge RAG indexes, searches, and organizes
# your documents. Every field is optional — sensible defaults are used
# when omitted.
#
# Quick start:
#   1. Copy this file → config.yaml
#   2. Set documents_dir to your folder
#   3. Run the server — everything else works out of the box
#
# Want a head start? Check the presets/ folder for domain-specific configs:
#   presets/cybersecurity.yaml  — Offensive/defensive security, CTFs, threat hunting
#   presets/developer.yaml      — Software engineering, APIs, DevOps
#   presets/research.yaml       — Academic research, papers, studies
#   presets/general.yaml        — Blank slate, zero domain logic
#
# Usage:  cp presets/developer.yaml config.yaml
#
# Need help? https://github.com/lyonzin/knowledge-rag


# ============================================================================
# PATHS
# ============================================================================
# Where your data lives. Relative paths resolve from this file's location.
# Absolute paths work too (e.g., /home/user/my-docs or C:\Users\you\docs).

paths:
  # Folder containing your documents (scanned recursively).
  # This is the only path most users need to change.
  documents_dir: "./documents"

  # Internal storage for indexes and embeddings.
  # You generally don't need to touch this.
  data_dir: "./data"

  # Persistent cache for embedding models (~250MB).
  # Prevents re-downloading after reboots (especially on Linux where /tmp is cleared).
  # Default: ./models_cache (relative to project root)
  models_cache_dir: "./models_cache"


# ============================================================================
# DOCUMENTS
# ============================================================================
# Control which files get indexed and how they're processed.

documents:
  # File types to index — file extensions plus exact filenames for
  # extensionless files (Dockerfile, Makefile, Tiltfile). Only matching
  # files are processed. Remove or comment out formats you don't need.
  supported_formats:
    - .md        # Markdown
    - .txt       # Plain text
    - .pdf       # PDF documents
    - .docx      # Word documents
    - .py        # Python source code
    - .c         # C source code
    - .h         # C/C++ header files
    - .cpp       # C++ source code
    - .js        # JavaScript
    - .jsx       # React JSX
    - .ts        # TypeScript
    - .tsx       # React TypeScript TSX
    - .json      # JSON files
    - .xml       # XML files
    - .go        # Go source code
    - .rs        # Rust source code
    - .kt        # Kotlin source code
    - .yaml      # YAML (Kubernetes manifests, configs)
    - .yml       # YAML (alternate extension)
    - .hujson    # HuJSON (JSON with comments and trailing commas)
    - .cue       # CUE configuration
    - .proto     # Protocol Buffers
    - .rego      # Open Policy Agent Rego
    - .sql       # SQL schemas and queries
    - .sh        # Shell scripts
    - .jq        # jq filters
    - Dockerfile # matched by exact filename (no extension)
    - Makefile   # matched by exact filename (no extension)
    - Tiltfile   # matched by exact filename (no extension)
    # - .xlsx    # Excel spreadsheets
    # - .pptx    # PowerPoint presentations
    # - .csv     # CSV data files
    # - .ipynb   # Jupyter Notebooks (extracts markdown + code cells)
    # - .mq4     # MetaTrader MQL4 source (opt-in, add to enable)
    # - .mqh     # MetaTrader MQL4/5 headers (opt-in, add to enable)

  # Exclude patterns — skip files/directories matching these patterns.
  # Uses fnmatch glob syntax. Patterns match against relative paths AND
  # individual path components (so "node_modules" matches at any depth).
  #
  # Examples:
  #   exclude_patterns:
  #     - "node_modules"     # Skip node_modules at any depth
  #     - ".git"             # Skip .git directories
  #     - "__pycache__"      # Skip Python cache
  #     - ".venv"            # Skip virtual environments
  #     - "*.tmp"            # Skip .tmp files anywhere
  #     - "drafts/*"         # Skip everything in top-level drafts/
  #
  # Default: [] (nothing excluded — all supported files are indexed)
  exclude_patterns: []

  # How documents are split into searchable chunks.
  #
  #   chunk_size    → Max characters per chunk. Larger = more context per
  #                   result, but less precise. Smaller = more precise,
  #                   but may lose context.
  #
  #   chunk_overlap → Characters shared between consecutive chunks.
  #                   Prevents information from being cut at chunk boundaries.
  #
  # Recommended presets:
  #   Short notes/snippets:  chunk_size: 500,  chunk_overlap: 100
  #   General use (default): chunk_size: 1000, chunk_overlap: 200
  #   Long technical docs:   chunk_size: 1500, chunk_overlap: 300
  chunking:
    chunk_size: 1000
    chunk_overlap: 200

  # ─────────────────────────────────────────────────────────────────────
  # Indexing performance tuning (v4.8.0+)
  # ─────────────────────────────────────────────────────────────────────
  #
  # batch_size — Number of chunks per ChromaDB batch add() call. Higher
  # values mean fewer SQLite round-trips and faster indexing, at the
  # cost of RAM (roughly batch_size * embedding_dim * 4 bytes for
  # float32 vectors sitting in memory while the batch flushes).
  # Default 500 preserves prior hardcoded behavior byte-for-byte.
  # Valid range: 1 to 5000 (values outside are clamped with a WARN).
  batch_size: 500

  # parallel_workers — Opt-in threading around ChromaDB batch adds.
  # Default 1 = single-threaded = safe on all platforms.
  #
  # When set > 1, per-batch ``collection.add(...)`` calls run inside a
  # ThreadPoolExecutor. ONNX embedding itself is serialized (internal
  # session lock), so the win comes from SQLite writes overlapping with
  # the NEXT batch's inference — NOT from parallel inference. Expect
  # modest gains (10-30%) on multi-batch documents, zero gain on
  # single-batch docs (the pool is skipped for those).
  #
  # WARNING: parallel_workers > 4 on Windows may hit ONNX threading
  # issues or SQLite lock contention — monitor stability. Recommended
  # cap: 4 on Windows, 8 on Linux/macOS.
  # Valid range: 1 to 16 (values outside are clamped with a WARN).
  parallel_workers: 1


# ============================================================================
# MODELS
# ============================================================================
# AI models used for understanding and ranking your documents.
# Defaults work well for most cases — only change if you know what you're doing.

models:
  # Embedding model — converts text into vectors for semantic search.
  # Runs locally via ONNX (no API key needed, no data leaves your machine).
  #
  # Two ways to configure (v4.8.0+):
  #
  # ---- Option A: named profile (recommended) ----
  # ``profile`` is a shorthand that fills model + dimensions + prefixes
  # from a single named preset. Available profiles:
  #
  #   compact       — BAAI/bge-small-en-v1.5 (384D, ~33MB, English-only, fastest — default)
  #   quality       — BAAI/bge-large-en-v1.5 (1024D, ~335MB, English-only, ~2x slower)
  #   multilingual  — intfloat/multilingual-e5-large (1024D, 100+ languages
  #                   incl. PT-BR / Spanish / Cyrillic / Arabic / CJK; ships
  #                   the required "query: " / "passage: " prefixes)
  #   custom        — opt out: respect the explicit model/dimensions/prefix
  #                   fields below (default when ``profile`` is omitted)
  #
  # When ``profile`` is anything other than "custom", the resolver IGNORES
  # any explicit ``model`` declared below and emits a
  # ``[WARN] profile takes precedence`` log on startup.
  #
  # ---- Option B: explicit model (custom profile) ----
  # Legacy shape — set ``profile: "custom"`` (or omit ``profile``) and
  # declare the model + dimensions directly. Additional community models:
  #   "BAAI/bge-base-en-v1.5"          → 768 dims, ~130MB, balanced
  #   "BAAI/bge-small-zh-v1.5"         → 384 dims, Chinese-only
  #   "intfloat/multilingual-e5-small" → 384 dims, 100+ languages (smaller e5)
  #
  # WARNING: changing profile / model / dimensions / prefix after indexing
  # REQUIRES a full reindex (``reindex_documents(force=True)``). Existing
  # chunks were embedded with the previous configuration and their vectors
  # are not comparable to the new ones.
  embedding:
    # Uncomment to switch to a named profile (see Option A above).
    # profile: "multilingual"

    # Explicit model / dimensions — respected only when profile is
    # "custom" (default). Ignored with a WARN under any other profile.
    model: "BAAI/bge-small-en-v1.5"
    dimensions: 384

    # Query / passage prefixes — prepended to text BEFORE embedding.
    # Required by some model families (intfloat/e5, thenlper/gte, ...);
    # leave empty for bge / mxbai. When a profile ships prefixes, this
    # override wins ONLY if the key is present here (empty string "" IS
    # an explicit override).
    query_prefix: ""
    passage_prefix: ""

    # GPU acceleration mode (v4.8.0+):
    #   "auto"  → probe CUDA at startup; use if ready, fall back to CPU (default)
    #   "true"  → force CUDA; if not ready, log WARN and fall back to CPU
    #   "false" → never probe; runs on CPU with zero startup overhead
    #
    # Legacy bool `gpu: true/false` continues to work — it is normalized to the
    # string form internally, so old configs keep behaving exactly the same.
    #
    # GPU dependency chain (all installed via pip in the same venv):
    #   1. onnxruntime-gpu, CUDA 12 variant (CUDA 13 is NOT supported by FastEmbed):
    #        pip install onnxruntime-gpu \
    #          --extra-index-url \
    #          https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
    #   2. NVIDIA runtime libs (pulled by pip, no system driver rewrite):
    #        pip install nvidia-cudnn-cu12 nvidia-cublas-cu12 nvidia-cuda-runtime-cu12 \
    #          nvidia-cufft-cu12 nvidia-curand-cu12 nvidia-cusolver-cu12 \
    #          nvidia-cusparse-cu12 nvidia-nvjitlink-cu12 nvidia-cuda-nvrtc-cu12
    #   3. Host: NVIDIA driver >= 525 (for CUDA 12). cuDNN 9 comes with nvidia-cudnn-cu12.
    #
    # Verify readiness manually without loading the embedding model:
    #   python -c "from mcp_server.server import FastEmbedEmbeddings; \
    #     import json; print(json.dumps(FastEmbedEmbeddings.verify_gpu_readiness().__dict__, \
    #     default=str, indent=2))"
    #
    # Full setup guide + troubleshooting: docs/gpu-setup.md
    gpu: "auto"

  # Reranker — re-scores search results for better accuracy.
  # Adds ~50-100ms per query but significantly improves result quality.
  # Set enabled: false on low-resource machines if search feels slow.
  reranker:
    enabled: true
    model: "Xenova/ms-marco-MiniLM-L-6-v2"
    # How many extra candidates to fetch before reranking.
    # Higher = better accuracy but slower. 3 is a good balance.
    top_k_multiplier: 3


# ============================================================================
# SEARCH
# ============================================================================
# Control search behavior and result limits.

search:
  # Number of results returned by default when no limit is specified.
  default_results: 5

  # Hard maximum — even if a client requests more, cap at this number.
  # Also bounds the candidate pool size in the semantic branch of the
  # hybrid retriever (``_do_semantic`` asks ChromaDB for
  # ``min(max_results * 3, config.max_results)`` candidates).
  #
  # v4.8.0 Fase 3 raised the default from 20 to 100. Prior to that fix,
  # the ``min(...)`` capped semantic candidates at 20 while BM25 pulled
  # up to ``max_results * 20 = 400`` — semantic silently starved on
  # hybrid mode. Callers passing ``max_results`` explicitly in the MCP
  # tool call are unaffected; only the default changed.
  max_results: 100

  # ChromaDB collection name. Change this to maintain separate
  # knowledge bases in the same data directory.
  # Example: "work_kb", "research_kb", "personal_kb"
  collection_name: "knowledge_base"

  # -----------------------------------------------------------------------
  # FTS5 Lexical Fast-Path (v4.8.2+, opt-in — default OFF pending v4.9.0)
  # -----------------------------------------------------------------------
  # Dedicated SQLite FTS5 index (data/fts5_index.db) optimized for exact
  # identifier queries (CVE-IDs, MITRE ATT&CK codes, VRT/CWE codes, file
  # hashes, error strings). See ADR-001 (storage), ADR-005 (tokenizer),
  # ADR-004 (v4.9.0 default flip gated on bench), ADR-009 (gate deferred
  # to CI perf-gate). Full user guide: docs/features/fts5_fast_path.md.
  #
  # The default remains `enabled: false` for v4.8.x. Setting `enabled: true`
  # here opts your instance in early — see the docs page for latency
  # expectations, migration behaviour, and telemetry counters.
  lexical_fast_path:
    # Master toggle. Default false preserves v4.8.1 behavior byte-for-byte.
    enabled: false

    # Minimum FTS5 hits required to skip the hybrid fallback. Queries
    # returning fewer than this many results fall back to hybrid so recall
    # is not sacrificed for a nearly-empty lexical result set.
    min_hits: 3

    # Run the cross-encoder reranker on FTS5 results. Default false (ADR-003)
    # because identifier queries rank correctly on bm25() alone and the
    # reranker adds ~40-80ms per query without measurable recall gain.
    rerank_enabled: false

    # First-match-wins regex patterns that classify a query as "lexical".
    # Ordering matters (PRD OQ-2). Default set targets bug bounty + SOC/DFIR
    # identifier vocabulary. Add custom project-specific codes below.
    patterns:
      - "[A-Z]{2,}-\\d+"        # H1-P4-XXX, MDR-AD002, CWE-79, MS17-010
      - "CVE-\\d{4}-\\d+"       # canonical CVE identifiers
      - "^[a-f0-9]{32,64}$"     # md5/sha1/sha256 file hashes


# ============================================================================
# CATEGORIES
# ============================================================================
# Organize documents into categories based on their folder path.
# When a document's path contains a key below, it gets tagged with
# that category — enabling filtered searches.
#
# How it works:
#   A file at documents/recipes/italian/pasta.md
#   matches the pattern "recipes/italian" → category "italian"
#   and also matches "recipes" → category "cooking"
#   (most specific match wins)
#
# Set to empty {} to disable auto-categorization entirely.
# Documents will still be searchable — just without category filters.
#
# ┌─────────────────────────────────────────────────────────────────────┐
# │  path pattern (in documents_dir)    │    category name              │
# ├─────────────────────────────────────┼───────────────────────────────┤
# │  "recipes/italian"                  │    "italian"                  │
# │  "recipes"                          │    "cooking"                  │
# │  "work/projects"                    │    "projects"                 │
# │  "journal"                          │    "personal"                 │
# └─────────────────────────────────────┴───────────────────────────────┘

category_mappings: {}


# ============================================================================
# KEYWORD ROUTING
# ============================================================================
# Route search queries to specific categories based on keywords.
# When a query contains any of these keywords, results from that
# category are prioritized.
#
# This is a PERFORMANCE optimization, not a filter — results from
# other categories still appear, just ranked lower.
#
# Set to empty {} for pure semantic search with no keyword bias.

keyword_routes: {}


# ============================================================================
# QUERY EXPANSION
# ============================================================================
# Expand search terms with synonyms and abbreviations.
# When someone searches for "JS", also search for "JavaScript".
# This improves BM25 (keyword) recall without affecting semantic search.
#
# Format:
#   term:
#     - synonym1
#     - synonym2
#
# Set to empty {} for no expansion (search terms used as-is).

query_expansions: {}

# Symmetric query expansion groups — every term expands to every other term
# in the same group. Use this when you want mutual synonyms without writing
# duplicate directional entries in query_expansions.
#
# Format:
#   - ["term a", "term b", "term c"]
#   - ["abbreviation", "full phrase"]
#
# The final expansion table is built from BOTH query_expansions and
# query_expansion_groups. Legacy query_expansions entries are kept, then group
# links are added on top.

query_expansion_groups: []


# ============================================================================
# ADVANCED
# ============================================================================
# Settings for power users. Most people never need to touch these.

# advanced:
#   # File watcher — auto-reindex when documents change.
#   # Disable if you prefer manual reindexing only.
#   watch_for_changes: true
#   watch_debounce_seconds: 5
#
#   # BM25 parameters (keyword search tuning).
#   # k1: term frequency saturation (1.2-2.0). Higher = more weight on repeated terms.
#   # b:  length normalization (0.0-1.0). Higher = penalizes long documents more.
#   # bm25_k1: 1.5
#   # bm25_b: 0.75
#
#   # Logging verbosity: DEBUG, INFO, WARNING, ERROR
#   # log_level: "INFO"


# ============================================================================
# SERVER (new in v4.0.0)
# ============================================================================
# Controls transport, networking, and enterprise features.
# All fields are optional — defaults preserve v3.x stdio behavior.

server:
  # Transport protocol: "stdio" (legacy), "sse", "streamable-http"
  # stdio: 1 process per client (compatible with all MCP clients)
  # sse: 1 server serves N clients over HTTP+SSE (recommended for multi-agent)
  # streamable-http: 1 server, HTTP streaming
  transport: "stdio"

  # Network settings (ignored when transport is stdio)
  host: "127.0.0.1"
  port: 8179

  # Auth: optional bearer token validation (SSE/HTTP only)
  auth:
    bearer_token: ""

  # Rate limiting: optional per-client request throttling
  rate_limit:
    enabled: false
    requests_per_minute: 60
    burst: 10

  # Metrics: optional Prometheus-compatible /metrics endpoint
  metrics:
    enabled: false
    port: 9179