Add ISCII and legacy-font Devanagari conversion

docs/legacy-indic-encoding.md

@@ -0,0 +1,85 @@
+# Legacy Indic encoding conversion
+
+OpenMed can convert Devanagari text stored as ISCII-1991 bytes or as a
+caller-described, ASCII-remapped legacy font before script routing and PII
+detection. Conversion is deterministic and local; it makes no network calls
+and does not log or persist source text.
+
+## ISCII conversion and offsets
+
+Pass raw bytes when source-byte offsets matter:
+
+```python
+from openmed.processing import iscii_to_unicode, unicode_to_iscii
+
+source = bytes.fromhex("cf cc e1 d5 20 d5 cf e8 cc da")
+conversion = iscii_to_unicode(source)
+
+assert conversion.text == "रमेश शर्मा"
+assert unicode_to_iscii(conversion.text) == source
+
+start = conversion.text.index("रमेश शर्मा")
+original_start, original_end = conversion.to_original_span(
+    start,
+    start + len("रमेश शर्मा"),
+)
+assert source[original_start:original_end] == source
+```
+
+The converter handles the ISCII Devanagari attribute sequence, standard
+nukta combinations, double danda, and the explicit and soft halant sequences.
+Output is placed in logical Unicode order, normalized to NFC, and aligned to
+the original byte stream. A Latin clinical note does not meet the conservative
+ISCII detection gate and is returned unchanged by `convert_legacy_encoding`.
+
+## User-supplied legacy-font maps
+
+OpenMed intentionally bundles no proprietary legacy-font mapping data. Supply
+a JSON or YAML file for a font whose mapping you are licensed to use:
+
+```json
+{
+  "name": "hospital-archive-font",
+  "provenance": "licensed by the data owner",
+  "mapping": {
+    "0x41": "र",
+    "B": "ा",
+    "67": "म"
+  }
+}
+```
+
+Keys may be byte integers, single-byte characters, decimal byte strings, or
+hexadecimal byte strings. Values are Unicode strings and may expand a single
+legacy glyph into multiple Unicode characters. Mapping files are limited to
+1 MiB and each replacement to 64 Unicode code points to bound untrusted input
+and expansion costs.
+
+```python
+from openmed.processing import LegacyFontMap, convert_legacy_encoding
+
+font_map = LegacyFontMap.from_file("hospital-archive-font.json")
+conversion = convert_legacy_encoding(
+    b"ABC",
+    encoding="legacy-font",
+    legacy_font_map=font_map,
+)
+assert conversion.text == "राम"
+```
+
+Auto-detection converts only dense runs with at least three mapped bytes and
+at least two resulting Devanagari letters. Use
+`encoding="legacy-font"` when a known, very short run is below that deliberately
+conservative threshold.
+
+## Provenance and limitations
+
+The built-in ISCII assignments follow the public Indian standard
+IS 13194:1991 and the Unicode Consortium's published ISCII semantics. The
+mapping table was constructed independently for OpenMed and is distributed
+under Apache-2.0. Caller-supplied legacy-font maps retain their own provenance
+and license; OpenMed does not redistribute them.
+
+Vedic stress extension sequences can be decoded, but complete Vedic Sanskrit
+accent and contextual invisible-letter fidelity is outside the byte-lossless
+guarantee. OCR and image processing are also outside this text-only feature.

mkdocs.yml

@@ -54,6 +54,7 @@ nav:
       - Configuration Profiles: profiles.md
       - Detector Plugin SDK: plugin-sdk.md
       - Batch Processing: batch-processing.md
+      - Legacy Indic Encoding: legacy-indic-encoding.md
       - Performance Profiling: profiling.md
       - Low-resource Benchmark: benchmarks/low-resource.md
       - Zero-shot NER How-to: zero-shot-howto.md

openmed/core/pipeline.py

@@ -906,6 +906,17 @@ def _normalize_code_mixed_tags(
         return tuple(normalized_tags)
 
     def stage1_normalize(self, text: str) -> NormalizedDocument:
+        from ..processing.legacy_encoding import convert_legacy_encoding
+
+        legacy_conversion = convert_legacy_encoding(text)
+        legacy_text = legacy_conversion.text
+        if legacy_conversion.encoding == "unicode":
+            legacy_origins = tuple((index, index + 1) for index in range(len(text)))
+        else:
+            # Auto-detected ISCII reaches this API as a Latin-1 surrogate
+            # string, so original byte and source-character offsets coincide.
+            legacy_origins = legacy_conversion.offset_map.converted_to_original_spans
+
         repair_encoding, encoding_repair_metadata = _encoding_repairer()
         # Encoding repair is an optional (ftfy) capability. When it is
         # unavailable the repairer is the identity function, so calling it once
@@ -920,7 +931,9 @@ def stage1_normalize(self, text: str) -> NormalizedDocument:
         normalized_to_original_span: list[tuple[int, int]] = []
         normalized_length = 0
 
-        for start, end, segment, is_whitespace in _iter_normalization_segments(text):
+        for start, end, segment, is_whitespace in _iter_normalization_segments(
+            legacy_text
+        ):
             normalized_start = normalized_length
             if is_whitespace:
                 normalized_segment = " "
@@ -935,14 +948,17 @@ def stage1_normalize(self, text: str) -> NormalizedDocument:
             if not normalized_segment:
                 continue
 
-            for index in range(start, end):
+            source_spans = legacy_origins[start:end]
+            source_start = min(span[0] for span in source_spans)
+            source_end = max(span[1] for span in source_spans)
+            for index in range(source_start, source_end):
                 original_to_normalized[index] = normalized_start
 
             normalized_parts.append(normalized_segment)
             normalized_length += len(normalized_segment)
             for _ in normalized_segment:
-                normalized_to_original.append(start)
-                normalized_to_original_span.append((start, end))
+                normalized_to_original.append(source_start)
+                normalized_to_original_span.append((source_start, source_end))
 
         normalized_text = "".join(normalized_parts)
         offset_map = OffsetMap(
@@ -960,6 +976,15 @@ def stage1_normalize(self, text: str) -> NormalizedDocument:
                 "original_length": len(text),
                 "normalized_length": len(normalized_text),
                 "encoding_repair": encoding_repair_metadata,
+                "legacy_encoding": {
+                    "encoding": legacy_conversion.encoding,
+                    "changed": legacy_conversion.changed,
+                    "converted_bytes": (
+                        sum(ord(char) >= 0x80 for char in text)
+                        if legacy_conversion.encoding == "iscii"
+                        else 0
+                    ),
+                },
             },
         )
         target_script = getattr(self.config, "chinese_target_script", None)

openmed/core/script_detect.py

@@ -23,10 +23,13 @@
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
-from typing import NoReturn
+from typing import TYPE_CHECKING, NoReturn
 
 from .language_pack_catalog import SCRIPT_LANGUAGE_HINTS
 
+if TYPE_CHECKING:
+    from ..processing.legacy_encoding import LegacyFontMap
+
 UNKNOWN_SCRIPT = "Unknown"
 
 logger = logging.getLogger(__name__)
@@ -519,6 +522,8 @@ class DetectionNormalization:
     folded_native_digits: int = 0
     indic_changes: int = 0
     indic_scripts: tuple[str, ...] = ()
+    converted_legacy_bytes: int = 0
+    legacy_encoding: str = "unicode"
     scripts: tuple[str, ...] = ()
     mixed_script: bool = False
     chinese_variant_normalized: bool = False
@@ -534,6 +539,7 @@ def changed(self) -> bool:
             or self.folded_confusables > 0
             or self.folded_native_digits > 0
             or self.indic_changes > 0
+            or self.converted_legacy_bytes > 0
             or self.chinese_variant_normalized
         )
 
@@ -565,6 +571,8 @@ def to_metadata(self) -> dict[str, object]:
             "folded_native_digits": self.folded_native_digits,
             "indic_changes": self.indic_changes,
             "indic_scripts": list(self.indic_scripts),
+            "converted_legacy_bytes": self.converted_legacy_bytes,
+            "legacy_encoding": self.legacy_encoding,
             "mixed_script": self.mixed_script,
             "opencc_available": self.opencc_available,
             "removed_zero_width": self.removed_zero_width,
@@ -1146,6 +1154,93 @@ def normalize_for_pii_detection(
     *,
     width_convention: str = "cjk",
     chinese_target_script: str | None = None,
+    legacy_font_map: LegacyFontMap | None = None,
+) -> DetectionNormalization:
+    """Convert legacy Indic text, then apply offset-safe Unicode defenses.
+
+    Likely ISCII-1991 input and caller-mapped legacy-font runs are converted
+    before script routing. The resulting offsets are composed back to the
+    original source-byte coordinate system; ordinary Unicode text follows the
+    existing normalization path unchanged.
+    """
+
+    from ..processing.legacy_encoding import convert_legacy_encoding
+
+    legacy_conversion = convert_legacy_encoding(
+        text,
+        legacy_font_map=legacy_font_map,
+    )
+    normalized = _normalize_unicode_for_pii_detection(
+        legacy_conversion.text,
+        width_convention=width_convention,
+        chinese_target_script=chinese_target_script,
+    )
+    if legacy_conversion.encoding == "unicode":
+        return normalized
+
+    legacy_origins = legacy_conversion.offset_map.converted_to_original_spans
+    starts: list[int] = []
+    ends: list[int] = []
+    for start, end in zip(normalized.offset_starts, normalized.offset_ends):
+        source_start, source_end = _source_span_for_legacy_range(
+            legacy_origins,
+            start,
+            end,
+        )
+        starts.append(source_start)
+        ends.append(source_end)
+
+    converted_sources: set[int] = set()
+    converted_by_span: dict[tuple[int, int], list[str]] = {}
+    for char, span in zip(legacy_conversion.text, legacy_origins):
+        converted_by_span.setdefault(span, []).append(char)
+    for (start, end), converted_chars in converted_by_span.items():
+        if text[start:end] != "".join(converted_chars):
+            converted_sources.update(range(start, end))
+
+    return DetectionNormalization(
+        text=normalized.text,
+        original_length=len(text),
+        offset_starts=tuple(starts),
+        offset_ends=tuple(ends),
+        removed_zero_width=normalized.removed_zero_width,
+        stripped_combining_marks=normalized.stripped_combining_marks,
+        folded_confusables=normalized.folded_confusables,
+        folded_native_digits=normalized.folded_native_digits,
+        indic_changes=normalized.indic_changes,
+        indic_scripts=normalized.indic_scripts,
+        converted_legacy_bytes=len(converted_sources),
+        legacy_encoding=legacy_conversion.encoding,
+        scripts=normalized.scripts,
+        mixed_script=normalized.mixed_script,
+        chinese_variant_normalized=normalized.chinese_variant_normalized,
+        chinese_target_script=normalized.chinese_target_script,
+        opencc_available=normalized.opencc_available,
+    )
+
+
+def _source_span_for_legacy_range(
+    origins: tuple[tuple[int, int], ...],
+    start: int,
+    end: int,
+) -> tuple[int, int]:
+    spans = origins[start:end]
+    if spans:
+        return min(span[0] for span in spans), max(span[1] for span in spans)
+    if start < len(origins):
+        anchor = origins[start][0]
+    elif origins:
+        anchor = origins[-1][1]
+    else:
+        anchor = 0
+    return anchor, anchor
+
+
+def _normalize_unicode_for_pii_detection(
+    text: str,
+    *,
+    width_convention: str = "cjk",
+    chinese_target_script: str | None = None,
 ) -> DetectionNormalization:
     """Fold adversarial Unicode artifacts while preserving offset remapping.
 
@@ -1240,11 +1335,11 @@ def normalize_for_pii_detection(
             and original_start > 0
             and _script_for_char(text[original_start - 1]) == "Ethiopic"
         )
-        if (
-            category == "Mn"
-            and _script_for_char(char) not in INDIC_SCRIPTS
-            and not attached_ethiopic_mark
-        ):
+        attached_indic_mark = category == "Mn" and _is_attached_indic_mark(
+            digit_folding.text,
+            index,
+        )
+        if category == "Mn" and not attached_indic_mark and not attached_ethiopic_mark:
             stripped_combining_marks += 1
             continue
 
@@ -1519,6 +1614,35 @@ def _script_for_char(char: str) -> str | None:
     return None
 
 
+def _is_attached_indic_mark(text: str, index: int) -> bool:
+    char = text[index]
+    if not _is_indic_codepoint(char) or index == 0:
+        return False
+    previous = text[index - 1]
+    return _is_indic_codepoint(previous) and unicodedata.category(previous)[0] in {
+        "L",
+        "M",
+    }
+
+
+def _is_indic_codepoint(char: str) -> bool:
+    codepoint = ord(char)
+    return any(
+        start <= codepoint <= end
+        for start, end in (
+            (0x0900, 0x097F),
+            (0x0980, 0x09FF),
+            (0x0A00, 0x0A7F),
+            (0x0A80, 0x0AFF),
+            (0x0B00, 0x0B7F),
+            (0x0B80, 0x0BFF),
+            (0x0C00, 0x0C7F),
+            (0x0C80, 0x0CFF),
+            (0x0D00, 0x0D7F),
+        )
+    )
+
+
 def _expand_detection_window(
     text: str,
     core_start: int,

openmed/processing/__init__.py

@@ -50,6 +50,16 @@
     deidentify_stream,
     replay,
 )
+from .legacy_encoding import (
+    ISCII_MAPPING_PROVENANCE,
+    ConversionOffsetMap,
+    LegacyConversion,
+    LegacyFontMap,
+    convert_legacy_encoding,
+    detect_legacy_encoding,
+    iscii_to_unicode,
+    unicode_to_iscii,
+)
 from .object_storage import (
     ObjectProgressCallback,
     ObjectStorageBatchResult,
@@ -64,6 +74,7 @@
     IndicNormalization,
     IndicNormalizer,
     TextProcessor,
+    normalize_indic_text,
     postprocess_text,
     preprocess_text,
 )
@@ -128,6 +139,15 @@
     "INDIC_SCRIPTS",
     "IndicNormalization",
     "IndicNormalizer",
+    "normalize_indic_text",
+    "ConversionOffsetMap",
+    "ISCII_MAPPING_PROVENANCE",
+    "LegacyConversion",
+    "LegacyFontMap",
+    "convert_legacy_encoding",
+    "detect_legacy_encoding",
+    "iscii_to_unicode",
+    "unicode_to_iscii",
     "preprocess_text",
     "postprocess_text",
     "TokenizationHelper",