diff --git a/bin/result-processor.py b/bin/result-processor.py index 68114c9..4dd103d 100755 --- a/bin/result-processor.py +++ b/bin/result-processor.py @@ -284,7 +284,7 @@ def validate_archive(archive): return 0 -def log_result_directory(result_directory, result_status): +def log_result_directory(result_directory, result_status, data=None): if myglobal.args.mode == "completion": # completion mode uses a very terse output data to feed into a # bash tab completion engine @@ -303,6 +303,10 @@ def log_result_directory(result_directory, result_status): myglobal.log.info("status: %s" % (result_status)) + if data is not None and data.get('partial'): + dropped_engines = data.get('dropped-engines', []) + myglobal.log.info("partial: yes (%d engine(s) dropped)" % (len(dropped_engines))) + return 0 @@ -420,7 +424,7 @@ def ls_result_directory(result_directory): myglobal.log.debug("result directory '%s' has no rickshaw-run" % (result_directory)) return 0 - log_result_directory(result_directory, status) + log_result_directory(result_directory, status, data) if myglobal.args.type == "short": pass @@ -691,7 +695,7 @@ def run_results_tag_mode(): data, status = load_rickshaw_run(run_dir) - log_result_directory(run_dir, status) + log_result_directory(run_dir, status, data) if data is not None: if myglobal.args.action == "ls": diff --git a/docs/how-benchmark-execution-works.md b/docs/how-benchmark-execution-works.md index d514775..5ee2876 100644 --- a/docs/how-benchmark-execution-works.md +++ b/docs/how-benchmark-execution-works.md @@ -480,6 +480,37 @@ no failures is considered passing. An iteration is considered failed if it doesn't have enough passing samples to meet the `num-samples` requirement. +### Partial runs and dropped engines + +A roadblock timeout anywhere in the run — during the per-sample +benchmark execution phases (`infra-start`, `server-start`, +`endpoint-start`, `client-start`, `client-stop`, `endpoint-stop`, +`server-stop`, `infra-stop`) or the post-benchmark phases +(stopping tools, sending data, endpoint cleanup) — causes +rickshaw to drop the missing engine(s) from the active follower +list, e.g. because one crashed (OOM kill) or lost its SSH +connection. What happens next depends on where the drop occurs: +a timeout during the per-sample phases aborts all remaining +samples and iterations for the rest of the run, while a timeout +during the post-benchmark phases only removes the missing +engine(s) and lets the run finish normally with the remaining +engines. Either way, the run can complete with fewer engines +than it started with, and without this feature that fact would +otherwise go unrecorded. + +When this happens, rickshaw logs a warning for each dropped +engine (naming the engine and the roadblock it missed) and an +end-of-run participation summary. The run's `rickshaw-run.json` +records `partial: true` and a `dropped-engines` list (each entry +has the engine ID and the roadblock it was dropped at); `crucible +ls`/`crucible get result` surface this as a `partial: yes (N +engine(s) dropped)` line alongside the normal `status` line. CDM +run documents (v9dev+) carry the same `partial`/`dropped-engines` +fields, so query results can be filtered or flagged accordingly. +Treat results from a partial run with caution — data from the +dropped engine(s) is missing or incomplete for the affected +phases. + ## Re-processing ### crucible postprocess diff --git a/docs/how-cdm-works.md b/docs/how-cdm-works.md index 0f01ab8..a8d32cd 100644 --- a/docs/how-cdm-works.md +++ b/docs/how-cdm-works.md @@ -41,7 +41,7 @@ run | Type | Purpose | Key fields | |------|---------|-----------| -| **run** | A single benchmark execution | run UUID, begin/end times, benchmark name, user, host | +| **run** | A single benchmark execution | run UUID, begin/end times, benchmark name, user, host, partial, dropped-engines | | **iteration** | One parameter combination | iteration UUID, primary metric, primary period, status | | **sample** | One execution of an iteration | sample UUID, status (pass/fail) | | **period** | A time window within a sample | period UUID, name, begin/end timestamps | @@ -413,7 +413,7 @@ CDM has evolved through several versions: |---------|------------| | v7dev | Original version, generic `id` fields | | v8dev | Document-specific UUID fields (`iteration-uuid`, `period-uuid`, etc.) | -| v9dev | Per-month index naming (`cdm-v9dev-metric_data@2026.06`), `metric_def` document type | +| v9dev | Per-month index naming (`cdm-v9dev-metric_data@2026.06`), `metric_def` document type, `partial`/`dropped-engines` fields on the run doc | | v10dev | `default-aggregation` field on metric_desc, per-metric aggregation control (sum/avg/max/min) | Multiple CDM versions can coexist in the same OpenSearch