Add conservative Indic morphology boundary refinement

openmed/core/decoding/__init__.py

@@ -17,10 +17,12 @@
     edge_f1,
 )
 from .spans import (
+    IndicSpanRefinement,
     TokenClassificationSpan,
     TokenClassificationStreamEvent,
     coerce_token_classification_spans,
     reconcile_stream_spans,
+    refine_indic_name_span,
     refine_privacy_filter_span,
     stable_span_id,
     stable_span_key,
@@ -43,6 +45,7 @@
     "EdgeDecisionTrace",
     "GraphExplainReport",
     "IncrementalViterbiState",
+    "IndicSpanRefinement",
     "SpanEdge",
     "SpanGraph",
     "SpanGraphConstraints",
@@ -57,6 +60,7 @@
     "edge_f1",
     "labels_to_token_spans",
     "reconcile_stream_spans",
+    "refine_indic_name_span",
     "refine_privacy_filter_span",
     "resolve_viterbi_biases",
     "stable_span_key",

openmed/core/decoding/spans.py

@@ -10,9 +10,12 @@
 
 import hashlib
 import re
+from collections.abc import Iterable
 from dataclasses import dataclass, replace
 from typing import Any, Final
 
+from openmed.core.labels import supports_name_boundary_refinement
+
 
 def trim_span_whitespace(start: int, end: int, text: str) -> tuple[int, int]:
     """Strip leading and trailing whitespace from ``text[start:end]``.
@@ -34,11 +37,189 @@ def trim_span_whitespace(start: int, end: int, text: str) -> tuple[int, int]:
 )
 
 
+@dataclass(frozen=True)
+class IndicSpanRefinement:
+    """An optional name-boundary refinement with absolute source offsets."""
+
+    original_start: int
+    original_end: int
+    start: int
+    end: int
+    applied: bool
+    reason: str
+    offset_map: tuple[tuple[int, int], ...]
+    grapheme_origins: tuple[tuple[int, int], ...]
+    rule: str | None = None
+
+    def to_source_span(self, start: int, end: int) -> tuple[int, int]:
+        """Map a span relative to the refined text back to source offsets."""
+        if not (0 <= start <= end <= len(self.offset_map)):
+            raise ValueError("span must fit within the refined output")
+        if start == end:
+            if start < len(self.offset_map):
+                anchor = self.offset_map[start][0]
+            elif self.offset_map:
+                anchor = self.offset_map[-1][1]
+            else:
+                anchor = self.start
+            return anchor, anchor
+        return self.offset_map[start][0], self.offset_map[end - 1][1]
+
+
+def refine_indic_name_span(
+    label: str,
+    start: int,
+    end: int,
+    text: str,
+    *,
+    enabled: bool = False,
+    language: str | None = None,
+    confidence: float = 0.0,
+    allowed_stems: Iterable[str] = (),
+    minimum_confidence: float = 0.9,
+    minimum_stem_graphemes: int = 2,
+) -> IndicSpanRefinement:
+    """Optionally tighten an Indic person-name span to an allow-listed stem.
+
+    The disabled path returns the input offsets exactly.  When enabled, the
+    span changes only if the label is name-like, confidence and allow-list
+    gates pass, and both the model span and the proposed end are grapheme
+    boundaries in the full source text.
+
+    Args:
+        label: Source entity label.
+        start: Inclusive source code-point offset.
+        end: Exclusive source code-point offset.
+        text: Full source text.
+        enabled: Opt-in switch; false preserves the original offsets.
+        language: Supported Indic language code or name.
+        confidence: Upstream entity confidence.
+        allowed_stems: Caller-approved exact name stems.
+        minimum_confidence: Minimum confidence required to refine.
+        minimum_stem_graphemes: Minimum graphemes retained in the name.
+
+    Returns:
+        An :class:`IndicSpanRefinement` carrying absolute offset mappings.
+    """
+    if not (0 <= start <= end <= len(text)):
+        raise ValueError("span must satisfy 0 <= start <= end <= len(text)")
+
+    unchanged = _indic_refinement_result(
+        text,
+        original_start=start,
+        original_end=end,
+        start=start,
+        end=end,
+        applied=False,
+        reason="disabled" if not enabled else "guard_rejected",
+    )
+    if not enabled:
+        return unchanged
+    if language is None or not supports_name_boundary_refinement(label):
+        return unchanged
+
+    # Import lazily to keep decoding helpers independent during processing
+    # package initialization.
+    from openmed.processing.morphology import grapheme_spans, stem_token
+
+    source_graphemes = grapheme_spans(text)
+    source_boundaries = {0, *(cluster_end for _, cluster_end in source_graphemes)}
+    if start not in source_boundaries or end not in source_boundaries:
+        return _indic_refinement_result(
+            text,
+            original_start=start,
+            original_end=end,
+            start=start,
+            end=end,
+            applied=False,
+            reason="unaligned_source_span",
+        )
+
+    analysis = stem_token(
+        text[start:end],
+        language,
+        confidence=confidence,
+        allowed_stems=allowed_stems,
+        minimum_confidence=minimum_confidence,
+        minimum_stem_graphemes=minimum_stem_graphemes,
+    )
+    if not analysis.applied:
+        return unchanged
+
+    local_start, local_end = analysis.stem_span
+    refined_start = start + local_start
+    refined_end = start + local_end
+    mapped = analysis.offset_map.to_source_span(0, len(analysis.stem))
+    if (
+        local_start != 0
+        or mapped != analysis.stem_span
+        or refined_start not in source_boundaries
+        or refined_end not in source_boundaries
+        or text[refined_start:refined_end] != analysis.stem
+    ):
+        return _indic_refinement_result(
+            text,
+            original_start=start,
+            original_end=end,
+            start=start,
+            end=end,
+            applied=False,
+            reason="offset_safety_rejected",
+        )
+
+    return _indic_refinement_result(
+        text,
+        original_start=start,
+        original_end=end,
+        start=refined_start,
+        end=refined_end,
+        applied=True,
+        reason="allowlisted_suffix",
+        rule=analysis.rule,
+    )
+
+
+def _indic_refinement_result(
+    text: str,
+    *,
+    original_start: int,
+    original_end: int,
+    start: int,
+    end: int,
+    applied: bool,
+    reason: str,
+    rule: str | None = None,
+) -> IndicSpanRefinement:
+    from openmed.processing.morphology import grapheme_spans
+
+    return IndicSpanRefinement(
+        original_start=original_start,
+        original_end=original_end,
+        start=start,
+        end=end,
+        applied=applied,
+        reason=reason,
+        offset_map=tuple((index, index + 1) for index in range(start, end)),
+        grapheme_origins=tuple(
+            (cluster_start, cluster_end)
+            for cluster_start, cluster_end in grapheme_spans(text)
+            if start <= cluster_start and cluster_end <= end
+        ),
+        rule=rule,
+    )
+
+
 def refine_privacy_filter_span(
     label: str,
     start: int,
     end: int,
     text: str,
+    *,
+    indic_morphology: bool = False,
+    language: str | None = None,
+    confidence: float = 0.0,
+    morphology_allowlist: Iterable[str] = (),
+    minimum_morphology_confidence: float = 0.9,
 ) -> tuple[int, int]:
     """Tighten obvious structured-PII spans when the model absorbs glue words.
 
@@ -62,7 +243,21 @@ def refine_privacy_filter_span(
         if span_text.lower().endswith(suffix):
             end -= len(suffix)
             break
-    return trim_span_whitespace(start, end, text)
+    start, end = trim_span_whitespace(start, end, text)
+    if indic_morphology:
+        refinement = refine_indic_name_span(
+            label,
+            start,
+            end,
+            text,
+            enabled=True,
+            language=language,
+            confidence=confidence,
+            allowed_stems=morphology_allowlist,
+            minimum_confidence=minimum_morphology_confidence,
+        )
+        return refinement.start, refinement.end
+    return start, end
 
 
 def _byte_offset(text: str, char_offset: int) -> int:
@@ -325,10 +520,12 @@ def stable_span_key(span: Any) -> tuple[int, int, str, str]:
 
 
 __all__ = [
+    "IndicSpanRefinement",
     "TokenClassificationSpan",
     "TokenClassificationStreamEvent",
     "coerce_token_classification_spans",
     "reconcile_stream_spans",
+    "refine_indic_name_span",
     "refine_privacy_filter_span",
     "stable_span_id",
     "stable_span_key",

openmed/core/labels.py

@@ -283,6 +283,13 @@
     }
 )
 
+# Boundary morphology is intentionally limited to labels that unambiguously
+# represent a person's name.  Prefixes and usernames are excluded because a
+# suffix-like surface may be part of the identifier itself.
+NAME_BOUNDARY_REFINEMENT_LABELS: Final[FrozenSet[str]] = frozenset(
+    {PERSON, FIRST_NAME, LAST_NAME, MIDDLE_NAME}
+)
+
 
 # ---------------------------------------------------------------------------
 # Policy metadata
@@ -1080,6 +1087,11 @@ def normalize_label(label: str, lang: str = "en") -> str:
     return OTHER
 
 
+def supports_name_boundary_refinement(label: str, lang: str = "en") -> bool:
+    """Return whether ``label`` is eligible for conservative name stemming."""
+    return normalize_label(label, lang=lang) in NAME_BOUNDARY_REFINEMENT_LABELS
+
+
 def id_subtype_for(label: str, lang: str = "en") -> str | None:
     """Return optional ID_NUM subtype metadata for a source label.
 
@@ -1120,7 +1132,9 @@ def hipaa_class_for(label: str, lang: str = "en") -> str:
 
 __all__ = [
     "CANONICAL_LABELS",
+    "NAME_BOUNDARY_REFINEMENT_LABELS",
     "normalize_label",
+    "supports_name_boundary_refinement",
     "id_subtype_for",
     "ID_ALIAS_SUBTYPES",
     "ID_SUBTYPES",