From 5fe1f4f8919b8197a31bc68843de79844b5270aa Mon Sep 17 00:00:00 2001 From: ibaldr89 <163388887+ibaldr89@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:40:51 +0000 Subject: [PATCH] fix(discover): detect extensionless scripts by shebang Signed-off-by: ibaldr89 <163388887+ibaldr89@users.noreply.github.com> --- src/discover/discover.c | 6 +- src/discover/discover.h | 14 ++ src/discover/language.c | 166 ++++++++++++++++++++++ tests/test_discover.c | 295 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 480 insertions(+), 1 deletion(-) diff --git a/src/discover/discover.c b/src/discover/discover.c index b2169f1aa..ec47c4862 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -651,7 +651,11 @@ static const char *file_skip_reason(const char *entry_name, const char *rel_path static CBMLanguage detect_file_language(const char *entry_name, const char *abs_path) { CBMLanguage lang = cbm_language_for_filename(entry_name); if (lang == CBM_LANG_COUNT) { - return CBM_LANG_COUNT; + /* Filename/extension detection failed: fall back to a conservative + * shebang probe so extensionless scripts get indexed (#1199). Filename + * detection stays authoritative — this runs only when it returns + * unknown. */ + return cbm_language_from_shebang(abs_path); } /* Special: .m files need content-based disambiguation */ const char *dot = strrchr(entry_name, '.'); diff --git a/src/discover/discover.h b/src/discover/discover.h index 87fc159d7..afb60fa36 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -50,6 +50,20 @@ CBMLanguage cbm_disambiguate_cls(const char *path); * On read failure, defaults to CBM_LANG_BITBAKE. */ CBMLanguage cbm_disambiguate_inc(const char *path); +/* Detect a supported script language from a file's shebang (#!...) first line. + * Conservative fallback used only when filename/extension detection is unknown + * (see detect_file_language); it never overrides extension or special-filename + * matches. Opens the file read-only and reads only a bounded first line. + * Recognizes the interpreter *basename* (not the parent path): + * python / python2 / python3 / dotted versions (python3.12) -> CBM_LANG_PYTHON + * sh / bash / dash / ksh / zsh -> CBM_LANG_BASH + * node / nodejs -> CBM_LANG_JAVASCRIPT + * ruby -> RUBY, perl -> PERL, php -> PHP, lua -> LUA + * Handles direct paths, "env ", "env -S ", and CRLF. + * Fails closed (returns CBM_LANG_COUNT) on read error, missing/malformed + * shebang, an embedded NUL in the first line, or an unknown interpreter. */ +CBMLanguage cbm_language_from_shebang(const char *path); + /* ── Gitignore pattern matching ──────────────────────────────────── */ typedef struct cbm_gitignore cbm_gitignore_t; diff --git a/src/discover/language.c b/src/discover/language.c index 4451b1b26..9854aa80c 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -945,6 +945,172 @@ const char *cbm_language_name(CBMLanguage lang) { return LANG_NAMES[lang] ? LANG_NAMES[lang] : "Unknown"; } +/* ── Shebang interpreter detection (extensionless scripts) ────────── */ + +/* Basename of an interpreter path: the segment after the last '/'. Shebangs + * are a POSIX convention, so only '/' is treated as a separator. */ +static const char *interp_basename(const char *path) { + const char *slash = strrchr(path, '/'); + return slash ? slash + SKIP_ONE : path; +} + +/* "python" optionally followed by an explicit numeric version (digits and dots + * only, e.g. "python3", "python3.12"). Bounded and explicit so arbitrary + * suffixes like "python-wrapper" are rejected. */ +static bool is_python_interp(const char *base) { + if (strncmp(base, "python", SLEN("python")) != 0) { + return false; + } + const char *version = base + SLEN("python"); + if (*version == '\0') { + return true; + } + + /* Each numeric component must contain at least one digit. */ + bool need_digit = true; + for (const char *v = version; *v; v++) { + if (isdigit((unsigned char)*v)) { + need_digit = false; + } else if (*v == '.' && !need_digit) { + need_digit = true; + } else { + return false; + } + } + return !need_digit; +} + +/* Map an interpreter basename to a language, or CBM_LANG_COUNT if unrecognized. + * Non-python interpreters are matched exactly (no prefix/suffix logic). */ +static CBMLanguage lang_for_interpreter(const char *base) { + if (is_python_interp(base)) { + return CBM_LANG_PYTHON; + } + static const struct { + const char *name; + CBMLanguage lang; + } INTERP_TABLE[] = { + {"sh", CBM_LANG_BASH}, {"bash", CBM_LANG_BASH}, {"dash", CBM_LANG_BASH}, + {"ksh", CBM_LANG_BASH}, {"zsh", CBM_LANG_BASH}, {"node", CBM_LANG_JAVASCRIPT}, + {"nodejs", CBM_LANG_JAVASCRIPT}, {"ruby", CBM_LANG_RUBY}, {"perl", CBM_LANG_PERL}, + {"php", CBM_LANG_PHP}, {"lua", CBM_LANG_LUA}, + }; + for (size_t i = 0; i < sizeof(INTERP_TABLE) / sizeof(INTERP_TABLE[0]); i++) { + if (strcmp(base, INTERP_TABLE[i].name) == 0) { + return INTERP_TABLE[i].lang; + } + } + return CBM_LANG_COUNT; +} + +/* Advance *cursor past leading blanks and return the next whitespace-delimited + * token (NUL-terminated in place), or NULL when the line is exhausted. */ +static char *shebang_next_token(char **cursor) { + char *p = *cursor; + while (*p == ' ' || *p == '\t') { + p++; + } + if (*p == '\0') { + *cursor = p; + return NULL; + } + char *start = p; + while (*p && *p != ' ' && *p != '\t') { + p++; + } + if (*p) { + *p = '\0'; + p++; + } + *cursor = p; + return start; +} + +CBMLanguage cbm_language_from_shebang(const char *path) { + if (!path) { + return CBM_LANG_COUNT; + } + + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return CBM_LANG_COUNT; /* fail closed on read error */ + } + + /* Read only a bounded first line. */ + char buf[CBM_SZ_256]; + size_t n = fread(buf, SKIP_ONE, sizeof(buf) - SKIP_ONE, f); + + /* Fail closed on any read error rather than parsing a partial buffer. */ + if (ferror(f)) { + (void)fclose(f); + return CBM_LANG_COUNT; + } + + /* If the bounded buffer filled without containing a newline, the first + * line may extend past our bound. Probe a single extra byte to tell an + * exact EOF (the whole file is <= 255 bytes) from a truncated longer + * line: any surviving byte -- including a newline just beyond the bound -- + * means the first line was cut off, so fail closed. A probe read error + * fails closed too. This keeps the read bounded (no unbounded line read + * or allocation). */ + bool have_newline = (memchr(buf, '\n', n) != NULL); + if (!have_newline && n == sizeof(buf) - SKIP_ONE) { + int probe = fgetc(f); + if (probe != EOF || ferror(f)) { + (void)fclose(f); + return CBM_LANG_COUNT; + } + } + (void)fclose(f); + + /* Must begin with "#!". */ + if (n < PAIR_LEN || buf[0] != '#' || buf[1] != '!') { + return CBM_LANG_COUNT; + } + + /* Isolate the first line; reject an embedded NUL before the newline. */ + size_t line_len = 0; + while (line_len < n && buf[line_len] != '\n') { + if (buf[line_len] == '\0') { + return CBM_LANG_COUNT; /* embedded NUL — treat as binary */ + } + line_len++; + } + /* Trim a trailing CR so CRLF first lines parse. */ + if (line_len > 0 && buf[line_len - SKIP_ONE] == '\r') { + line_len--; + } + buf[line_len] = '\0'; + + /* First token after "#!" is the interpreter (or env). */ + char *cursor = buf + PAIR_LEN; + char *interp = shebang_next_token(&cursor); + if (!interp) { + return CBM_LANG_COUNT; + } + const char *base = interp_basename(interp); + + /* "env [-S] [args...]": the real interpreter is the next token. + * Only the plain "env " and "env -S/--split-string [args]" + * shapes are supported. After the optional -S, the interpreter token must + * be a real command, so reject option tokens (leading '-') and NAME=value + * assignments (containing '=') -- e.g. "env PYTHON=/usr/bin/python + * python-wrapper", where env would treat the first token as an env-var + * setting rather than the program to run. */ + if (strcmp(base, "env") == 0) { + char *tok = shebang_next_token(&cursor); + if (tok && (strcmp(tok, "-S") == 0 || strcmp(tok, "--split-string") == 0)) { + tok = shebang_next_token(&cursor); + } + if (!tok || tok[0] == '-' || strchr(tok, '=') != NULL) { + return CBM_LANG_COUNT; + } + base = interp_basename(tok); + } + + return lang_for_interpreter(base); +} + /* ── .m file disambiguation ──────────────────────────────────────── */ /* Simple substring search helper */ diff --git a/tests/test_discover.c b/tests/test_discover.c index 57cbefe75..2cabaa28a 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -395,6 +395,47 @@ TEST(discover_bounded_count_fails_closed_after_deadline) { PASS(); } +/* The allocation-free bounded count must apply the same shebang fallback as + * full discovery (issue #1199): extensionless scripts count, extensionless + * plain text does not, and the limit boundary stays exact over that mix. */ +TEST(discover_bounded_count_matches_shebang_discovery) { + char *base = th_mktempdir("cbm_disc_bounded_sb"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "keep.c"), "int keep;\n"); + th_write_file(TH_PATH(base, "bin/build"), "#!/usr/bin/env python3\nprint(1)\n"); + th_write_file(TH_PATH(base, "bin/deploy"), "#!/bin/bash\necho hi\n"); + th_write_file(TH_PATH(base, "bin/notes"), "just some plain text\nno shebang here\n"); + + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int count = 0; + int full_rc = cbm_discover(base, &opts, &files, &count); + bool has_build = discover_has_rel_path(files, count, "bin/build"); + bool has_deploy = discover_has_rel_path(files, count, "bin/deploy"); + bool has_notes = discover_has_rel_path(files, count, "bin/notes"); + cbm_discover_free(files, count); + + int exact_count = -1; + cbm_discover_status_t exact = + cbm_discover_count_bounded(base, &opts, 3, cbm_now_ms() + 2000, &exact_count); + int limited_count = -1; + cbm_discover_status_t limited = + cbm_discover_count_bounded(base, &opts, 2, cbm_now_ms() + 2000, &limited_count); + + th_cleanup(base); + + ASSERT_EQ(full_rc, 0); + ASSERT_TRUE(has_build); + ASSERT_TRUE(has_deploy); + ASSERT_FALSE(has_notes); + ASSERT_EQ(count, 3); + ASSERT_EQ(exact, CBM_DISCOVER_OK); + ASSERT_EQ(exact_count, 3); + ASSERT_EQ(limited, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT_EQ(limited_count, 2); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1356,6 +1397,239 @@ TEST(discover_many_nested_gitignores_do_not_exhaust_matcher_ownership) { PASS(); } +/* ── Shebang fallback for extensionless scripts (issue #1199) ────── */ + +/* Language detected for a discovered file by relative path, or CBM_LANG_COUNT + * if the file was not indexed. */ +static CBMLanguage discover_lang_of(const cbm_file_info_t *files, int count, const char *rel) { + for (int i = 0; i < count; i++) { + if (strcmp(files[i].rel_path, rel) == 0) { + return files[i].language; + } + } + return CBM_LANG_COUNT; +} + +/* Write raw bytes (may contain embedded NUL) to base/rel. Parent must exist. */ +static int write_bytes_file(const char *path, const void *data, size_t n) { + FILE *f = cbm_fopen(path, "wb"); + if (!f) { + return -1; + } + size_t w = fwrite(data, 1, n, f); + fclose(f); + return w == n ? 0 : -1; +} + +/* Discover a single file with the given first-line content and report the + * language it was classified as via *out_lang (CBM_LANG_COUNT if unindexed). + * Returns true only when temp-dir creation, the file write, and discovery all + * succeeded, so callers can assert setup success before asserting the language + * -- otherwise a negative expectation (CBM_LANG_COUNT) could pass vacuously + * when setup silently failed. */ +static bool shebang_probe(const char *prefix, const char *rel, const char *content, + CBMLanguage *out_lang) { + *out_lang = CBM_LANG_COUNT; + char *base = th_mktempdir(prefix); + if (!base) { + return false; + } + bool ok = false; + if (th_write_file(TH_PATH(base, rel), content) == 0) { + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + if (cbm_discover(base, &opts, &files, &count) == 0) { + *out_lang = discover_lang_of(files, count, rel); + ok = true; + } + cbm_discover_free(files, count); + } + th_cleanup(base); + return ok; +} + +TEST(shebang_direct_python_path) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_dpy", "bin/run", "#!/usr/bin/python3\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_env_python) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_epy", "bin/run", "#!/usr/bin/env python3\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_env_split_string_python_args) { + CBMLanguage lang; + ASSERT( + shebang_probe("cbm_sb_spy", "bin/run", "#!/usr/bin/env -S python3 -u\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_versioned_python) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_vpy", "bin/run", "#!/usr/bin/python3.12\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +TEST(shebang_bash_direct) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_bash", "bin/run", "#!/bin/bash\necho hi\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_BASH); + PASS(); +} + +TEST(shebang_env_node_javascript) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_node", "bin/run", "#!/usr/bin/env node\nconsole.log(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_JAVASCRIPT); + PASS(); +} + +TEST(shebang_crlf_first_line) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_crlf", "bin/run", "#!/usr/bin/env python3\r\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +/* Recognized extension stays authoritative even with a conflicting shebang. */ +TEST(shebang_extension_authoritative) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_ext", "weird.py", "#!/bin/bash\nprint(1)\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_PYTHON); + PASS(); +} + +/* Special filename stays authoritative even with a conflicting shebang. */ +TEST(shebang_special_filename_authoritative) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_mk", "Makefile", "#!/usr/bin/env python3\nall:\n\techo\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_MAKEFILE); + PASS(); +} + +TEST(shebang_plain_text_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_txt", "notes", "just some plain text\nno shebang here\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +TEST(shebang_malformed_unindexed) { + /* Missing '!' — not a shebang. */ + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_mal", "notes", "#/usr/bin/python3\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +TEST(shebang_unknown_interpreter_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_unk", "notes", "#!/usr/bin/frobnicate\ndata\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* Guard against arbitrary prefix matches (python-wrapper must NOT map). */ +TEST(shebang_prefix_not_matched_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_pw", "notes", "#!/usr/bin/python-wrapper\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* Numeric version components must be non-empty (reject trailing/doubled dots). */ +TEST(shebang_malformed_python_version_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_pdot", "notes", "#!/usr/bin/python3.\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + ASSERT(shebang_probe("cbm_sb_pddot", "notes", "#!/usr/bin/python3..12\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* env NAME=value assignment (with a slash in the value) must NOT be mistaken + * for an interpreter: env would set the var, not run "python" (issue #1199, + * Finding 2). */ +TEST(shebang_env_assignment_not_matched_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_asn", "notes", + "#!/usr/bin/env PYTHON=/usr/bin/python python-wrapper\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + /* Same shape behind -S must also stay unindexed. */ + ASSERT(shebang_probe("cbm_sb_asn2", "notes", + "#!/usr/bin/env -S PYTHON=/usr/bin/python python\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +/* Unsupported env option forms (a leading '-' token that is not -S/ + * --split-string) must stay unindexed rather than be treated as the + * interpreter (issue #1199, Finding 2). */ +TEST(shebang_env_unsupported_option_unindexed) { + CBMLanguage lang; + ASSERT(shebang_probe("cbm_sb_eopt", "notes", "#!/usr/bin/env -i python3\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + ASSERT(shebang_probe("cbm_sb_eopt2", "notes", "#!/usr/bin/env -S -i python3\nx\n", &lang)); + ASSERT_EQ(lang, CBM_LANG_COUNT); + PASS(); +} + +TEST(shebang_embedded_nul_unindexed) { + char *base = th_mktempdir("cbm_sb_nul"); + ASSERT(base != NULL); + /* NUL embedded within the first line -> fail closed. */ + static const char bytes[] = "#!/usr/bin/py\0thon3\nprint(1)\n"; + ASSERT_EQ(write_bytes_file(TH_PATH(base, "run"), bytes, sizeof(bytes) - 1), 0); + + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), 0); + ASSERT_FALSE(discover_has_rel_path(files, count, "run")); + + cbm_discover_free(files, count); + th_cleanup(base); + PASS(); +} + +/* A first line longer than the bounded read window (255 bytes) with no + * newline inside that window must fail closed even when it begins with an + * otherwise-valid Python shebang: the interpreter token is truncated, so the + * file must remain unindexed (issue #1199, Finding 1). */ +TEST(shebang_oversized_first_line_unindexed) { + char *base = th_mktempdir("cbm_sb_long"); + ASSERT(base != NULL); + + /* "#!/usr/bin/python3 " + padding to push the newline past byte 255. */ + char content[512]; + int off = snprintf(content, sizeof(content), "#!/usr/bin/python3 "); + ASSERT(off > 0 && off < (int)sizeof(content)); + for (int i = off; i < 400; i++) { + content[i] = 'a'; + } + content[400] = '\n'; + content[401] = '\0'; + ASSERT_EQ(write_bytes_file(TH_PATH(base, "run"), content, 401), 0); + + cbm_discover_opts_t opts = {0}; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), 0); + ASSERT_FALSE(discover_has_rel_path(files, count, "run")); + + cbm_discover_free(files, count); + th_cleanup(base); + PASS(); +} + /* ── Suite ─────────────────────────────────────────────────────── */ SUITE(discover) { @@ -1427,11 +1701,32 @@ SUITE(discover) { RUN_TEST(pattern_stories); RUN_TEST(pattern_dts_full); + /* Shebang fallback for extensionless scripts (issue #1199) */ + RUN_TEST(shebang_direct_python_path); + RUN_TEST(shebang_env_python); + RUN_TEST(shebang_env_split_string_python_args); + RUN_TEST(shebang_versioned_python); + RUN_TEST(shebang_bash_direct); + RUN_TEST(shebang_env_node_javascript); + RUN_TEST(shebang_crlf_first_line); + RUN_TEST(shebang_extension_authoritative); + RUN_TEST(shebang_special_filename_authoritative); + RUN_TEST(shebang_plain_text_unindexed); + RUN_TEST(shebang_malformed_unindexed); + RUN_TEST(shebang_unknown_interpreter_unindexed); + RUN_TEST(shebang_prefix_not_matched_unindexed); + RUN_TEST(shebang_malformed_python_version_unindexed); + RUN_TEST(shebang_env_assignment_not_matched_unindexed); + RUN_TEST(shebang_env_unsupported_option_unindexed); + RUN_TEST(shebang_embedded_nul_unindexed); + RUN_TEST(shebang_oversized_first_line_unindexed); + /* Integration tests (cross-platform) */ RUN_TEST(discover_simple); RUN_TEST(discover_wide_sibling_fanout_exceeds_initial_walk_stack); RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); + RUN_TEST(discover_bounded_count_matches_shebang_discovery); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore);