From 7b2c3c49ac8ac641a7b59c1ff56af139314bd6af Mon Sep 17 00:00:00 2001 From: Shashwat Mishra Date: Tue, 4 Aug 2026 21:59:59 +0530 Subject: [PATCH] fix: quarantine files on EXIT_NONZERO instead of aborting chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a worker process exits with a nonzero code (e.g. internal parse-limit abort on a pathological file), treat it the same as CRASH/HANG: quarantine the offending file via the existing two-consecutive-strikes mechanism and continue indexing the rest of the chunk. Previously, EXIT_NONZERO caused the supervisor to abort the entire chunk, losing all indexed data for that directory. This is impactful for large-scale repos (Android AOSP: 253GB, 60M+ files) where machine-generated files routinely trigger internal limits. The two-consecutive-strikes intersection mechanism already guards against false quarantines — a systemic nonzero exit (e.g. bad CLI arg) produces no recurring suspect, so the intersection is empty and the supervisor gives up (same behavior as before). Tested on Android QSSI17 (253GB, 24 chunks) + QCM6490 vendor (244 chunks). Result: 11.35M nodes indexed, 0 data loss, 2 genuinely pathological files quarantined (checkpatch.pl, a 140K-line generated .ts file). Signed-off-by: Shashwat Mishra --- src/mcp/mcp.c | 26 +++++++++++++++++++++----- src/pipeline/pass_definitions.c | 5 +++-- src/pipeline/pass_parallel.c | 5 +++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index d40571b30..b2c6c957d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -7607,13 +7607,29 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; /* good files indexed; quarantined files reported as crash/hang */ } - if (wr2.outcome == CBM_PROC_CRASH || wr2.outcome == CBM_PROC_HANG) { + if (wr2.outcome == CBM_PROC_CRASH || wr2.outcome == CBM_PROC_HANG || + wr2.outcome == CBM_PROC_EXIT_NONZERO) { last_outcome = wr2.outcome; cbm_index_worker_result_free(&wr2); - /* crash vs hang: the phase this file is quarantined under and - * reported as in skipped[]. A fault signal → "crash"; a - * no-progress kill → "hang". */ - const char *phase = (last_outcome == CBM_PROC_HANG) ? "hang" : "crash"; + /* crash vs hang vs nonzero-exit: the phase this file is quarantined + * under and reported as in skipped[]. A fault signal → "crash"; a + * no-progress kill → "hang"; a graceful nonzero exit (e.g. an + * internal parse-limit/abort on a pathological file) → "error". + * EXIT_NONZERO is attributed via the SAME marker-journal suspect + * mechanism as a crash: the two-consecutive-strikes intersection + * below still guards against quarantining an innocent file, and a + * SYSTEMIC nonzero exit (e.g. a bad arg) produces no recurring + * suspect → the intersection is empty → give_up (correct). This + * makes a single pathological file skip-and-continue instead of + * aborting the whole chunk. */ + const char *phase; + if (last_outcome == CBM_PROC_HANG) { + phase = "hang"; + } else if (last_outcome == CBM_PROC_EXIT_NONZERO) { + phase = "error"; + } else { + phase = "crash"; + } int sus_n = 0; char **suspects = supervisor_read_suspects(marker_path, &sus_n); (void)remove(marker_path); /* fresh journal for the next re-run */ diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index ebbb95414..ee8414c32 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -535,8 +535,9 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t if (!phase) { phase = "crash"; } - const char *reason = - (strcmp(phase, "hang") == 0) ? "quarantined after hang" : "quarantined after crash"; + const char *reason = (strcmp(phase, "hang") == 0) ? "quarantined after hang" + : (strcmp(phase, "error") == 0) ? "quarantined after error" + : "quarantined after crash"; cbm_pipeline_add_file_error(ctx->pipeline, rel, reason, phase); errors++; continue; diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 21b9561b3..95dccc8a6 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -841,8 +841,9 @@ static void extract_worker(int worker_id, void *ctx_ptr) { if (!phase) { phase = "crash"; } - const char *reason = - (strcmp(phase, "hang") == 0) ? "quarantined after hang" : "quarantined after crash"; + const char *reason = (strcmp(phase, "hang") == 0) ? "quarantined after hang" + : (strcmp(phase, "error") == 0) ? "quarantined after error" + : "quarantined after crash"; pp_err_add(errs, fi->rel_path, reason, phase); ws->errors++; continue;