feat(search): index prose content for BM25 full-text search

Section nodes (markdown) and Module nodes (YAML/JSON) previously exposed
only their heading/name to BM25, so search_graph could not match the prose
body or a config description. Index that text so content is searchable.

- store: add a `body` column to the nodes_fts FTS5 table; new
  cbm_store_fts_rebuild() drops+recreates the table (upgrading legacy
  4-column databases) and backfills `body` from each node's docstring,
  guarded by json_valid() against malformed-JSON rows
- pipeline: both FTS backfill sites now call cbm_store_fts_rebuild()
- mcp: stop excluding Section/Module from BM25 results (they rank below
  code symbols, so existing result ordering is preserved)
- internal/cbm: capture the markdown section body beneath each heading
  (#518) and promote top-level description/summary/purpose values onto
  the file's Module node (#519), reusing the existing docstring property
- tests: 7 extraction cases + 3 store FTS cases

Closes #518
Closes #519

Signed-off-by: ShauryaaSharma <shauryasofficial27@gmail.com>

Author: ShauryaaSharma

internal/cbm/extract_defs.c

@@ -2834,7 +2834,8 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
 // --- Class definition extraction ---
 
 // Push a simple class definition (used by config language extractors).
-static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label) {
+static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, const char *label,
+                                  const char *docstring) {
     CBMArena *a = ctx->arena;
     CBMDefinition def;
     memset(&def, 0, sizeof(def));
@@ -2845,6 +2846,7 @@ static void push_simple_class_def(CBMExtractCtx *ctx, TSNode node, char *name, c
     def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET;
     def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET;
     def.is_exported = true;
+    def.docstring = docstring; // section body for Markdown (#518); NULL otherwise
     cbm_defs_push(&ctx->result->defs, a, def);
 }
 
@@ -2943,6 +2945,57 @@ static char *extract_markdown_heading_name(CBMArena *a, TSNode node, const char
     return trim_heading_name(name);
 }
 
+// Max bytes of Markdown section body captured for BM25 content search (#518).
+enum { CBM_MD_SECTION_BODY_MAX = 500 };
+
+// Capture the prose body beneath a Markdown heading so BM25 can search the
+// content, not just the heading text (#518). In the tree-sitter-markdown grammar
+// each heading lives inside a `section` node that also holds the body blocks and
+// any nested subsections; the body is the source span between the heading and
+// either the first nested subsection or the end of the section. Returns NULL when
+// there is no enclosing section or no body text. The result is trimmed and capped
+// at CBM_MD_SECTION_BODY_MAX bytes (without splitting a UTF-8 sequence).
+static char *extract_markdown_section_body(CBMArena *a, TSNode heading, const char *source) {
+    TSNode parent = ts_node_parent(heading);
+    if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "section") != 0) {
+        return NULL;
+    }
+    uint32_t body_start = ts_node_end_byte(heading);
+    uint32_t body_end = ts_node_end_byte(parent);
+    // Stop at the first nested subsection — it gets its own Section node + body.
+    uint32_t cc = ts_node_child_count(parent);
+    for (uint32_t i = 0; i < cc; i++) {
+        TSNode ch = ts_node_child(parent, i);
+        if (ts_node_start_byte(ch) >= body_start && strcmp(ts_node_type(ch), "section") == 0) {
+            body_end = ts_node_start_byte(ch);
+            break;
+        }
+    }
+    // Trim surrounding whitespace/newlines (UTF-8 lead/continuation bytes are all
+    // >= 0x80, so a byte-wise <= ' ' test never cuts a multi-byte character).
+    while (body_start < body_end && (unsigned char)source[body_start] <= ' ') {
+        body_start++;
+    }
+    while (body_end > body_start && (unsigned char)source[body_end - 1] <= ' ') {
+        body_end--;
+    }
+    if (body_end <= body_start) {
+        return NULL;
+    }
+    size_t len = (size_t)(body_end - body_start);
+    if (len > CBM_MD_SECTION_BODY_MAX) {
+        len = CBM_MD_SECTION_BODY_MAX;
+        // Back off so the cap never splits a UTF-8 multi-byte sequence.
+        while (len > 0 && ((unsigned char)source[body_start + len] & 0xC0) == 0x80) {
+            len--;
+        }
+        if (len == 0) {
+            return NULL;
+        }
+    }
+    return cbm_arena_strndup(a, source + body_start, len);
+}
+
 // INI: extract section name from section node.
 static char *find_ini_section_name(CBMArena *a, TSNode node, const char *source) {
     uint32_t nc = ts_node_child_count(node);
@@ -2996,6 +3049,7 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
     CBMArena *a = ctx->arena;
     char *name = NULL;
     const char *label = "Class";
+    const char *docstring = NULL;
 
     if (ctx->language == CBM_LANG_TOML &&
         (strcmp(kind, "table") == 0 || strcmp(kind, "table_array_element") == 0)) {
@@ -3008,14 +3062,15 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char
                (strcmp(kind, "atx_heading") == 0 || strcmp(kind, "setext_heading") == 0)) {
         name = extract_markdown_heading_name(a, node, kind, ctx->source);
         label = "Section";
+        docstring = extract_markdown_section_body(a, node, ctx->source); // #518
     } else if (ctx->language == CBM_LANG_HCL && strcmp(kind, "block") == 0) {
         name = find_hcl_block_name(a, node, ctx->source);
     } else {
         return false;
     }
 
     if (name && name[0]) {
-        push_simple_class_def(ctx, node, name, label);
+        push_simple_class_def(ctx, node, name, label, docstring);
     }
     return true;
 }
@@ -5514,6 +5569,174 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
     }
 }
 
+// ── Config module description promotion (#519) ──────────────────────────────
+// YAML/JSON metadata files (META.yaml, skill manifests, …) carry their most
+// search-relevant text in a top-level `description`/`summary`/`purpose` value.
+// That value is otherwise dropped — only the key becomes a Variable node — so
+// BM25 can't find a module by its description. We promote the value onto the
+// file's Module node docstring, which the FTS `body` column then indexes.
+
+// Case-insensitive ASCII equality (keys are conventionally lowercase, but accept
+// "Description"/"SUMMARY" too).
+static bool cfg_ci_eq(const char *x, const char *y) {
+    while (*x && *y) {
+        char cx = *x;
+        char cy = *y;
+        if (cx >= 'A' && cx <= 'Z') {
+            cx = (char)(cx + ('a' - 'A'));
+        }
+        if (cy >= 'A' && cy <= 'Z') {
+            cy = (char)(cy + ('a' - 'A'));
+        }
+        if (cx != cy) {
+            return false;
+        }
+        x++;
+        y++;
+    }
+    return *x == '\0' && *y == '\0';
+}
+
+static bool cfg_is_desc_key(const char *key) {
+    return key && (cfg_ci_eq(key, "description") || cfg_ci_eq(key, "summary") ||
+                   cfg_ci_eq(key, "purpose"));
+}
+
+// Trim surrounding whitespace and strip one layer of matching quotes, in place.
+static char *cfg_strip_quotes_trim(char *s) {
+    if (!s) {
+        return NULL;
+    }
+    while (*s && (unsigned char)*s <= ' ') {
+        s++;
+    }
+    size_t len = strlen(s);
+    while (len > 0 && (unsigned char)s[len - 1] <= ' ') {
+        s[--len] = '\0';
+    }
+    if (len >= 2 && ((s[0] == '"' && s[len - 1] == '"') || (s[0] == '\'' && s[len - 1] == '\''))) {
+        s[len - 1] = '\0';
+        s++;
+    }
+    return s;
+}
+
+// Copy at most `cap` bytes of `s`, never splitting a UTF-8 sequence; NULL if empty.
+static char *cfg_arena_capped(CBMArena *a, const char *s, size_t cap) {
+    if (!s || !s[0]) {
+        return NULL;
+    }
+    size_t len = strlen(s);
+    if (len > cap) {
+        len = cap;
+        while (len > 0 && ((unsigned char)s[len] & 0xC0) == 0x80) {
+            len--;
+        }
+        if (len == 0) {
+            return NULL;
+        }
+    }
+    return cbm_arena_strndup(a, s, len);
+}
+
+// Descend stream/document/block_node to the top-level YAML block_mapping.
+static TSNode cfg_find_yaml_mapping(TSNode root) {
+    TSNode none = {0};
+    TSNode cur = root;
+    for (int depth = 0; depth < 8; depth++) {
+        uint32_t n = ts_node_child_count(cur);
+        TSNode next = none;
+        bool have_next = false;
+        for (uint32_t i = 0; i < n; i++) {
+            TSNode ch = ts_node_child(cur, i);
+            const char *ck = ts_node_type(ch);
+            if (strcmp(ck, "block_mapping") == 0) {
+                return ch;
+            }
+            if (!have_next && (strcmp(ck, "stream") == 0 || strcmp(ck, "document") == 0 ||
+                               strcmp(ck, "block_node") == 0)) {
+                next = ch;
+                have_next = true;
+            }
+        }
+        if (!have_next) {
+            break;
+        }
+        cur = next;
+    }
+    return none;
+}
+
+// Descend document wrappers to the top-level JSON object.
+static TSNode cfg_find_json_object(TSNode root) {
+    TSNode none = {0};
+    TSNode cur = root;
+    for (int depth = 0; depth < 6; depth++) {
+        if (strcmp(ts_node_type(cur), "object") == 0) {
+            return cur;
+        }
+        uint32_t n = ts_node_child_count(cur);
+        TSNode next = none;
+        bool have_next = false;
+        for (uint32_t i = 0; i < n; i++) {
+            TSNode ch = ts_node_child(cur, i);
+            const char *ck = ts_node_type(ch);
+            if (strcmp(ck, "object") == 0) {
+                return ch;
+            }
+            if (!have_next && strcmp(ck, "document") == 0) {
+                next = ch;
+                have_next = true;
+            }
+        }
+        if (!have_next) {
+            break;
+        }
+        cur = next;
+    }
+    return none;
+}
+
+// Find a top-level description/summary/purpose value in a YAML or JSON file and
+// return it (trimmed, unquoted, capped) for promotion to the Module docstring.
+static const char *extract_config_module_description(CBMExtractCtx *ctx) {
+    if (ctx->language != CBM_LANG_YAML && ctx->language != CBM_LANG_JSON) {
+        return NULL;
+    }
+    CBMArena *a = ctx->arena;
+    TSNode mapping;
+    const char *pair_kind;
+    if (ctx->language == CBM_LANG_YAML) {
+        mapping = cfg_find_yaml_mapping(ctx->root);
+        pair_kind = "block_mapping_pair";
+    } else {
+        mapping = cfg_find_json_object(ctx->root);
+        pair_kind = "pair";
+    }
+    if (ts_node_is_null(mapping)) {
+        return NULL;
+    }
+    uint32_t n = ts_node_named_child_count(mapping);
+    for (uint32_t i = 0; i < n; i++) {
+        TSNode pair = ts_node_named_child(mapping, i);
+        if (strcmp(ts_node_type(pair), pair_kind) != 0) {
+            continue;
+        }
+        TSNode key = ts_node_child_by_field_name(pair, TS_FIELD("key"));
+        TSNode val = ts_node_child_by_field_name(pair, TS_FIELD("value"));
+        if (ts_node_is_null(key) || ts_node_is_null(val)) {
+            continue;
+        }
+        const char *key_txt = cfg_strip_quotes_trim(cbm_node_text(a, key, ctx->source));
+        if (!cfg_is_desc_key(key_txt)) {
+            continue;
+        }
+        char *val_txt = cfg_strip_quotes_trim(cbm_node_text(a, val, ctx->source));
+        return cfg_arena_capped(a, val_txt, CBM_MD_SECTION_BODY_MAX);
+    }
+    return NULL;
+}
+
 void cbm_extract_definitions(CBMExtractCtx *ctx) {
     const CBMLangSpec *spec = cbm_lang_spec(ctx->language);
     if (!spec) {
@@ -5533,6 +5756,8 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {
     mod.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET;
     mod.is_exported = true;
     mod.is_test = ctx->result->is_test_file;
+    // Promote a YAML/JSON top-level description onto the Module for BM25 (#519).
+    mod.docstring = extract_config_module_description(ctx);
     cbm_defs_push(&ctx->result->defs, a, mod);
 
     // Walk AST for function/class definitions

src/mcp/mcp.c

@@ -1334,7 +1334,11 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
         ") fts "
         "JOIN nodes n ON n.id = fts.rowid "
         "WHERE n.project = ?2 "
-        "  AND n.label NOT IN ('File','Folder','Module','Section','Variable','Project') "
+        /* Section and Module are searchable (#518/#519): their FTS `body` column
+         * carries markdown section prose and YAML/JSON description text.  They
+         * rank below code symbols (no boost above), so code results stay first.
+         * File/Folder/Variable/Project remain excluded as structural noise. */
+        "  AND n.label NOT IN ('File','Folder','Variable','Project') "
         "  AND (?6 IS NULL OR n.file_path LIKE ?6) "
         "ORDER BY rank "
         "LIMIT ?3 OFFSET ?4";
@@ -1359,17 +1363,17 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
      * Uses the identical subquery structure so the FTS5 early-exit applies here too. */
     int total = 0;
     {
-        const char *count_sql =
-            "SELECT COUNT(*) FROM ("
-            "    SELECT fts.rowid FROM ("
-            "        SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
-            "        ORDER BY bm25(nodes_fts) LIMIT ?3"
-            "    ) fts "
-            "    JOIN nodes n ON n.id = fts.rowid "
-            "    WHERE n.project = ?2 "
-            "      AND n.label NOT IN ('File','Folder','Module','Section','Variable','Project')"
-            "      AND (?6 IS NULL OR n.file_path LIKE ?6)"
-            ")";
+        const char *count_sql = "SELECT COUNT(*) FROM ("
+                                "    SELECT fts.rowid FROM ("
+                                "        SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
+                                "        ORDER BY bm25(nodes_fts) LIMIT ?3"
+                                "    ) fts "
+                                "    JOIN nodes n ON n.id = fts.rowid "
+                                "    WHERE n.project = ?2 "
+                                /* Mirror the label filter in the main query above (#518/#519). */
+                                "      AND n.label NOT IN ('File','Folder','Variable','Project')"
+                                "      AND (?6 IS NULL OR n.file_path LIKE ?6)"
+                                ")";
         sqlite3_stmt *cs = NULL;
         if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {
             sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,

src/pipeline/pipeline.c

@@ -877,20 +877,11 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
             }
         }
 
-        /* FTS5 backfill: populate nodes_fts with camelCase-split names.
-         * Contentless FTS5 requires the special 'delete-all' command instead of
-         * DELETE FROM to wipe prior rows (there's no underlying content table).
-         * Falls back to plain names if cbm_camel_split is unavailable (which
-         * shouldn't happen because we always register it, but we stay defensive). */
-        cbm_store_exec(hash_store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");
-        if (cbm_store_exec(hash_store,
-                           "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
-                           "SELECT id, cbm_camel_split(name), qualified_name, label, file_path "
-                           "FROM nodes;") != CBM_STORE_OK) {
-            cbm_store_exec(hash_store,
-                           "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
-                           "SELECT id, name, qualified_name, label, file_path FROM nodes;");
-        }
+        /* FTS5 backfill: rebuild nodes_fts with camelCase-split names and prose
+         * bodies (markdown sections, YAML/JSON descriptions, docstrings) so BM25
+         * matches content as well as identifiers (#518/#519).  cbm_store_fts_rebuild
+         * also upgrades legacy 4-column tables to the schema carrying `body`. */
+        cbm_store_fts_rebuild(hash_store);
 
         cbm_store_close(hash_store);
         cbm_log_info("pass.timing", "pass", "persist_hashes", "files", itoa_buf(file_count));

src/pipeline/pipeline_incremental.c

@@ -649,17 +649,9 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *
 
         /* FTS5 rebuild after incremental dump.  The btree dump path bypasses
          * any triggers that could have kept nodes_fts synchronized, so we
-         * rebuild from the nodes table here.  See the full-dump path in
-         * pipeline.c for the matching logic. */
-        cbm_store_exec(hash_store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");
-        if (cbm_store_exec(hash_store,
-                           "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
-                           "SELECT id, cbm_camel_split(name), qualified_name, label, file_path "
-                           "FROM nodes;") != CBM_STORE_OK) {
-            cbm_store_exec(hash_store,
-                           "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) "
-                           "SELECT id, name, qualified_name, label, file_path FROM nodes;");
-        }
+         * rebuild from the nodes table here (also indexing prose bodies for
+         * content search — see cbm_store_fts_rebuild and #518/#519). */
+        cbm_store_fts_rebuild(hash_store);
 
         cbm_store_close(hash_store);
     }