diff --git a/nemo/collections/asr/parts/utils/eou_utils.py b/nemo/collections/asr/parts/utils/eou_utils.py index 256e8b418951..fb81aeb19e0f 100644 --- a/nemo/collections/asr/parts/utils/eou_utils.py +++ b/nemo/collections/asr/parts/utils/eou_utils.py @@ -173,8 +173,12 @@ def evaluate_eou( false_negatives += len(reference) - r_idx missing += len(reference) - r_idx - missing -= len(earlycut_ids) # Remove the references that were missed due to early cutoff - false_negatives -= len(earlycut_ids) # Remove the references that were missed due to early cutoff + # Only references counted by the block above can be discounted here. An early cutoff + # that r_idx already advanced past was never added, so subtracting it drives both + # counts negative. + counted_earlycut = sum(1 for idx in earlycut_ids if idx >= r_idx) + missing -= counted_earlycut # Remove the references that were missed due to early cutoff + false_negatives -= counted_earlycut # Remove the references that were missed due to early cutoff return EOUResult( latency=latency, early_cutoff=early_cutoff, diff --git a/tests/collections/asr/test_asr_eou.py b/tests/collections/asr/test_asr_eou.py index 5446f2ac5eb5..927118b88674 100644 --- a/tests/collections/asr/test_asr_eou.py +++ b/tests/collections/asr/test_asr_eou.py @@ -185,3 +185,32 @@ def test_cal_eou_metrics_from_frame_labels_multiple_eou(self): assert eou_metrics.missing == 0 assert eou_metrics.early_cutoff == [] assert np.allclose(eou_metrics.latency, [delay] * len(ref_eou_times)) + + +class TestEvaluateEOUCounts: + def test_counts_stay_non_negative_when_early_cutoff_is_skipped(self): + """An early cutoff that r_idx advances past is never counted, so it must not be subtracted.""" + reference = [ + {"start_time": 0.0, "end_time": 2.0}, + {"start_time": 5.0, "end_time": 7.0}, + ] + prediction = [ + {"start_time": 0.0, "end_time": 1.0, "eou_prob": 0.9}, + {"start_time": 5.5, "end_time": 7.0, "eou_prob": 0.9}, + ] + + result = evaluate_eou(prediction=prediction, reference=reference, threshold=None, collar=0.1) + + assert result.false_negatives == 0 + assert result.missing == 0 + + def test_trailing_early_cutoff_is_still_discounted(self): + """A trailing early cutoff is counted by the tail, so the discount must still apply.""" + reference = [{"start_time": 0.0, "end_time": 2.0}] + prediction = [{"start_time": 0.0, "end_time": 1.0, "eou_prob": 0.9}] + + result = evaluate_eou(prediction=prediction, reference=reference, threshold=None, collar=0.1) + + assert result.false_negatives == 0 + assert result.missing == 0 + assert len(result.early_cutoff) == 1