diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index d891e7dd9..02ecddae3 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -8293,16 +8293,6 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { /* ── search_code v2: graph-augmented code search ─────────────── */ -/* Strip non-ASCII bytes to guarantee valid UTF-8 JSON output */ -enum { ASCII_MAX = 127 }; -static void sanitize_ascii(char *s) { - for (unsigned char *p = (unsigned char *)s; *p; p++) { - if (*p > ASCII_MAX) { - *p = '?'; - } - } -} - /* Intermediate grep match */ typedef struct { char file[CBM_SZ_512]; @@ -8503,8 +8493,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } char *source = read_file_lines(abs_path, s, e); if (source) { - sanitize_ascii(source); - yyjson_mut_obj_add_strcpy(doc, item, "source", source); + char *safe_source = sanitize_utf8_lossy(source); + if (safe_source) { + yyjson_mut_obj_add_strcpy(doc, item, "source", safe_source); + free(safe_source); + } free(source); if (truncated) { yyjson_mut_obj_add_int(doc, item, "source_start", s); @@ -8519,8 +8512,11 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } char *ctx = read_file_lines(abs_path, ctx_start, ctx_end); if (ctx) { - sanitize_ascii(ctx); - yyjson_mut_obj_add_strcpy(doc, item, "context", ctx); + char *safe_context = sanitize_utf8_lossy(ctx); + if (safe_context) { + yyjson_mut_obj_add_strcpy(doc, item, "context", safe_context); + free(safe_context); + } yyjson_mut_obj_add_int(doc, item, "context_start", ctx_start); free(ctx); } @@ -8687,16 +8683,17 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc build_dedup_files_array(doc, sr, output_count, raw, raw_count)); } else { /* json-stringified tree: cols + column-ordered row arrays. FULL mode - * appends a per-row object cell with the (guarded, windowed) source — - * attach_result_source semantics unchanged. */ + * appends a per-row object cell with the (guarded, windowed) source; + * context requests append the corresponding context object. */ + bool attach_context = context_lines > 0 && mode != MODE_FULL; yyjson_mut_val *jcols = yyjson_mut_arr(doc); static const char *const sc_cols[] = {"qn", "label", "file", "lines", "matches", "in", "out"}; for (size_t ci = 0; ci < sizeof(sc_cols) / sizeof(sc_cols[0]); ci++) { yyjson_mut_arr_add_str(doc, jcols, sc_cols[ci]); } - if (mode == MODE_FULL) { - yyjson_mut_arr_add_str(doc, jcols, "source"); + if (mode == MODE_FULL || attach_context) { + yyjson_mut_arr_add_str(doc, jcols, mode == MODE_FULL ? "source" : "context"); } yyjson_mut_obj_add_val(doc, root_obj, "cols", jcols); @@ -8722,7 +8719,7 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc yyjson_mut_arr_add_val(row, ml); yyjson_mut_arr_add_int(doc, row, r->in_degree); yyjson_mut_arr_add_int(doc, row, r->out_degree); - if (mode == MODE_FULL) { + if (mode == MODE_FULL || attach_context) { yyjson_mut_val *src = yyjson_mut_obj(doc); attach_result_source(doc, src, r, mode, context_lines, root_path); yyjson_mut_arr_add_val(row, src); @@ -8789,7 +8786,11 @@ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_matc char *json = yy_doc_to_str(doc); if (json) { - sanitize_ascii(json); + char *safe_json = sanitize_utf8_lossy(json); + if (safe_json) { + free(json); + json = safe_json; + } } yyjson_mut_doc_free(doc); @@ -8860,8 +8861,10 @@ static grep_match_t *collect_grep_matches(FILE *fp, const char *root_path, size_ safe_grow(gm, gm_count, gm_cap, PAIR_LEN); snprintf(gm[gm_count].file, sizeof(gm[0].file), "%s", file); gm[gm_count].line = (int)strtol(sep1 + SKIP_ONE, NULL, CBM_DECIMAL_BASE); - snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", sep2 + SKIP_ONE); - sanitize_ascii(gm[gm_count].content); + char *safe_content = sanitize_utf8_lossy(sep2 + SKIP_ONE); + snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", + safe_content ? safe_content : sep2 + SKIP_ONE); + free(safe_content); gm_count++; } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index a56aebdf5..f0c22e30b 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -3654,6 +3654,250 @@ TEST(search_code_multi_word) { PASS(); } +/* Regression guard: search_code full results must preserve valid UTF-8 source. */ +static bool is_valid_json_response(const char *json); + +TEST(search_code_full_preserves_utf8_source) { + char tmp[512]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_srch_utf8_XXXXXX"); + ASSERT_TRUE(cbm_mkdtemp(tmp) != NULL); + + char project_dir[640]; + snprintf(project_dir, sizeof(project_dir), "%s/project", tmp); + ASSERT_EQ(cbm_mkdir(project_dir), 0); + char design_dir[768]; + snprintf(design_dir, sizeof(design_dir), "%s/design", project_dir); + ASSERT_EQ(cbm_mkdir(design_dir), 0); + + char source_path[768]; + snprintf(source_path, sizeof(source_path), "%s/design.md", design_dir); + FILE *fp = cbm_fopen(source_path, "wb"); + ASSERT_NOT_NULL(fp); + const char source[] = "# accounting-design\nРусский текст: бухгалтерский учет.\n"; + ASSERT_EQ(fwrite(source, 1, sizeof(source) - SKIP_ONE, fp), sizeof(source) - SKIP_ONE); + ASSERT_EQ(fclose(fp), 0); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + const char *project = "utf8-search"; + cbm_mcp_server_set_project(srv, project); + cbm_store_upsert_project(st, project, project_dir); + + cbm_node_t section = {.project = project, + .label = "Section", + .name = "accounting-design", + .qualified_name = "utf8-search.design.accounting-design", + .file_path = "design/design.md", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(st, §ion), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":97,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"utf8-search\",\"pattern\":\"accounting-design\"," + "\"file_pattern\":\"*.md\",\"path_filter\":\"^design/\"," + "\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows"); + ASSERT_NOT_NULL(rows); + ASSERT_TRUE(yyjson_arr_size(rows) > 0); + yyjson_val *row = yyjson_arr_get(rows, 0); + yyjson_val *source_obj = yyjson_arr_get(row, 7); + yyjson_val *source_val = yyjson_obj_get(source_obj, "source"); + ASSERT_NOT_NULL(source_val); + ASSERT_STR_EQ(yyjson_get_str(source_val), source); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cbm_unlink(source_path); + cbm_rmdir(design_dir); + cbm_rmdir(project_dir); + cbm_rmdir(tmp); + PASS(); +} + +TEST(search_code_raw_match_preserves_utf8_content) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + char raw_path[512]; + snprintf(raw_path, sizeof(raw_path), "%s/project/raw.md", tmp); + const char raw_source[] = "header\nraw-Русский content\n"; + FILE *fp = cbm_fopen(raw_path, "wb"); + ASSERT_NOT_NULL(fp); + ASSERT_EQ(fwrite(raw_source, 1, sizeof(raw_source) - SKIP_ONE, fp), + sizeof(raw_source) - SKIP_ONE); + ASSERT_EQ(fclose(fp), 0); + + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_node_t node = {.project = "test-project", + .label = "Section", + .name = "raw", + .qualified_name = "test-project.raw", + .file_path = "raw.md", + .start_line = 1, + .end_line = 1}; + ASSERT_GT(cbm_store_upsert_node(st, &node), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":98,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"test-project\",\"pattern\":\"raw-\"," + "\"file_pattern\":\"*.md\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *raw_obj = yyjson_obj_get(yyjson_doc_get_root(doc), "raw_matches"); + ASSERT_NOT_NULL(raw_obj); + yyjson_val *raw_rows = yyjson_obj_get(raw_obj, "rows"); + ASSERT_NOT_NULL(raw_rows); + ASSERT_TRUE(yyjson_arr_size(raw_rows) > 0); + yyjson_val *raw_row = yyjson_arr_get(raw_rows, 0); + ASSERT_STR_EQ(yyjson_get_str(yyjson_arr_get(raw_row, 2)), "raw-Русский content"); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +TEST(search_code_context_preserves_utf8_context) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + char context_path[512]; + snprintf(context_path, sizeof(context_path), "%s/project/context.md", tmp); + const char context_source[] = "before-до\ncontext-needle\nпосле-после\n"; + FILE *fp = cbm_fopen(context_path, "wb"); + ASSERT_NOT_NULL(fp); + ASSERT_EQ(fwrite(context_source, 1, sizeof(context_source) - SKIP_ONE, fp), + sizeof(context_source) - SKIP_ONE); + ASSERT_EQ(fclose(fp), 0); + + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_node_t node = {.project = "test-project", + .label = "Section", + .name = "context", + .qualified_name = "test-project.context", + .file_path = "context.md", + .start_line = 1, + .end_line = 3}; + ASSERT_GT(cbm_store_upsert_node(st, &node), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":99,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"test-project\",\"pattern\":\"context-needle\"," + "\"format\":\"json\",\"context\":1,\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *cols = yyjson_obj_get(root, "cols"); + yyjson_val *rows = yyjson_obj_get(root, "rows"); + ASSERT_NOT_NULL(cols); + ASSERT_NOT_NULL(rows); + ASSERT_TRUE(yyjson_arr_size(rows) > 0); + + size_t context_index = SIZE_MAX; + size_t col_count = yyjson_arr_size(cols); + for (size_t i = 0; i < col_count; i++) { + const char *col = yyjson_get_str(yyjson_arr_get(cols, i)); + if (col && strcmp(col, "context") == 0) { + context_index = i; + break; + } + } + ASSERT_TRUE(context_index != SIZE_MAX); + yyjson_val *context_obj = yyjson_arr_get(yyjson_arr_get(rows, 0), context_index); + ASSERT_NOT_NULL(context_obj); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(context_obj, "context")), context_source); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(context_obj, "context_start")), 1); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +TEST(search_code_invalid_utf8_still_returns_valid_json) { + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + char invalid_path[512]; + snprintf(invalid_path, sizeof(invalid_path), "%s/project/invalid.md", tmp); + static const unsigned char invalid_source[] = { + 'i', 'n', 'v', 'a', 'l', 'i', 'd', '-', 'n', 'e', 'e', 'd', 'l', 'e', '\n', + 'c', 'o', 'n', 't', 'e', 'x', 't', ' ', 0xFF, '\n', + }; + FILE *fp = cbm_fopen(invalid_path, "wb"); + ASSERT_NOT_NULL(fp); + ASSERT_EQ(fwrite(invalid_source, 1, sizeof(invalid_source), fp), sizeof(invalid_source)); + ASSERT_EQ(fclose(fp), 0); + + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + cbm_node_t node = {.project = "test-project", + .label = "Section", + .name = "invalid", + .qualified_name = "test-project.invalid", + .file_path = "invalid.md", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(st, &node), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":100,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_code\",\"arguments\":{" + "\"project\":\"test-project\",\"pattern\":\"invalid-needle\"," + "\"mode\":\"full\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_TRUE(is_valid_json_response(inner)); + yyjson_doc *doc = yyjson_read(inner, strlen(inner), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *rows = yyjson_obj_get(yyjson_doc_get_root(doc), "rows"); + ASSERT_NOT_NULL(rows); + ASSERT_TRUE(yyjson_arr_size(rows) > 0); + yyjson_val *source_obj = yyjson_arr_get(yyjson_arr_get(rows, 0), 7); + ASSERT_NOT_NULL(source_obj); + const char *safe_source = yyjson_get_str(yyjson_obj_get(source_obj, "source")); + ASSERT_NOT_NULL(safe_source); + ASSERT_NOT_NULL(strstr(safe_source, "\xEF\xBF\xBD")); + ASSERT_NULL(memchr(safe_source, 0xFF, strlen(safe_source))); + yyjson_doc_free(doc); + + free(inner); + free(resp); + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + /* Reproduce-first (#687): scoped content search over a repo whose ROOT PATH * contains a space. write_scoped_filelist emits "/" records that the * Unix pipeline pipes to grep via xargs. With plain `xargs` (newline-split) the @@ -9622,6 +9866,10 @@ SUITE(mcp) { RUN_TEST(tool_search_code_missing_pattern); RUN_TEST(tool_search_code_no_project); RUN_TEST(search_code_multi_word); + RUN_TEST(search_code_full_preserves_utf8_source); + RUN_TEST(search_code_raw_match_preserves_utf8_content); + RUN_TEST(search_code_context_preserves_utf8_context); + RUN_TEST(search_code_invalid_utf8_still_returns_valid_json); RUN_TEST(search_code_scoped_path_with_spaces_issue687); #ifdef _WIN32 RUN_TEST(search_code_scoped_path_with_cjk_root_issue903);