Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions nemo/collections/asr/parts/utils/eou_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,12 @@ def evaluate_eou(
false_negatives += len(reference) - r_idx
missing += len(reference) - r_idx

missing -= len(earlycut_ids) # Remove the references that were missed due to early cutoff
false_negatives -= len(earlycut_ids) # Remove the references that were missed due to early cutoff
# Only references counted by the block above can be discounted here. An early cutoff
# that r_idx already advanced past was never added, so subtracting it drives both
# counts negative.
counted_earlycut = sum(1 for idx in earlycut_ids if idx >= r_idx)
missing -= counted_earlycut # Remove the references that were missed due to early cutoff
false_negatives -= counted_earlycut # Remove the references that were missed due to early cutoff
return EOUResult(
latency=latency,
early_cutoff=early_cutoff,
Expand Down
30 changes: 30 additions & 0 deletions tests/collections/asr/test_asr_eou.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from nemo.collections.asr.parts.utils.eou_utils import (
EOUResult,
cal_eou_metrics_from_frame_labels,
evaluate_eou,
get_SegLST_from_frame_labels,
)

Expand Down Expand Up @@ -138,3 +139,32 @@ def test_cal_eou_metrics_from_frame_labels_multiple_eou(self):
assert eou_metrics.missing == 0
assert eou_metrics.early_cutoff == []
assert np.allclose(eou_metrics.latency, [delay] * len(ref_eou_times))


class TestEvaluateEOUCounts:
def test_counts_stay_non_negative_when_early_cutoff_is_skipped(self):
"""An early cutoff that r_idx advances past is never counted, so it must not be subtracted."""
reference = [
{"start_time": 0.0, "end_time": 2.0},
{"start_time": 5.0, "end_time": 7.0},
]
prediction = [
{"start_time": 0.0, "end_time": 1.0, "eou_prob": 0.9},
{"start_time": 5.5, "end_time": 7.0, "eou_prob": 0.9},
]

result = evaluate_eou(prediction=prediction, reference=reference, threshold=None, collar=0.1)

assert result.false_negatives == 0
assert result.missing == 0

def test_trailing_early_cutoff_is_still_discounted(self):
"""A trailing early cutoff is counted by the tail, so the discount must still apply."""
reference = [{"start_time": 0.0, "end_time": 2.0}]
prediction = [{"start_time": 0.0, "end_time": 1.0, "eou_prob": 0.9}]

result = evaluate_eou(prediction=prediction, reference=reference, threshold=None, collar=0.1)

assert result.false_negatives == 0
assert result.missing == 0
assert len(result.early_cutoff) == 1
Loading