From b862b69bfe64bb0fa0a9155afec16f7c933a7742 Mon Sep 17 00:00:00 2001 From: Maziyar Panahi Date: Sun, 19 Jul 2026 17:23:53 +0200 Subject: [PATCH] feat: add Central and Eastern European PII packs --- README.md | 8 +- clients/go/client.go | 3 + clients/typescript/src/index.ts | 5 +- docs/anonymization.md | 4 +- docs/api/openapi.json | 30 ++- docs/faq.md | 6 +- docs/feature-map.md | 2 +- docs/index.md | 4 +- docs/languages.md | 30 +++ docs/rest-recipes.md | 7 +- docs/troubleshooting.md | 8 +- docs/website/assets/script.js | 2 +- docs/website/index.html | 20 +- models.jsonl | 10 +- openmed/core/anonymizer/engine.py | 2 + openmed/core/anonymizer/locales.py | 2 + openmed/core/anonymizer/providers/__init__.py | 4 + .../core/anonymizer/providers/clinical_ids.py | 27 +++ .../core/anonymizer/providers/registry_ids.py | 9 + openmed/core/anonymizer/registry.py | 3 + openmed/core/model_card.py | 2 +- openmed/core/pii_i18n.py | 196 +++++++++++++++++- openmed/core/script_detect.py | 6 +- openmed/eval/golden/fixtures/i18n/uk.jsonl | 1 + openmed/service/client.py | 3 + openmed/service/schemas.py | 3 + openmed/training/synthetic/locale_phi.py | 39 ++++ tests/unit/core/test_id_provider_registry.py | 2 + tests/unit/core/test_script_detect.py | 6 + tests/unit/test_pii_i18n.py | 145 ++++++++++++- .../training/test_synthetic_locale_phi.py | 10 +- 31 files changed, 548 insertions(+), 51 deletions(-) create mode 100644 openmed/eval/golden/fixtures/i18n/uk.jsonl diff --git a/README.md b/README.md index b57818b5d..18f7ddc12 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ No cloud. No vendor lock-in. No patient data leaving your network.

- 2,000+ models  ·  17 model-backed PII languages  ·  600+ PII checkpoints  ·  100% on-device  ·  Apache-2.0 + 2,000+ models  ·  20 model-backed PII languages  ·  600+ PII checkpoints  ·  100% on-device  ·  Apache-2.0

@@ -503,10 +503,10 @@ On non-Apple-Silicon hosts, MLX model names are automatically substituted with t --- -## Multilingual PII (17 model-backed languages) +## Multilingual PII (20 model-backed languages) -Extraction and de-identification support **17 supported PII language codes**: -`ar`, `de`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, `pt`, `ro`, `te`, `th`, and `tr`, with **600+ PII checkpoints** in total. +Extraction and de-identification support **20 supported PII language codes**: +`ar`, `cs`, `de`, `el`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, `pt`, `ro`, `te`, `th`, `tr`, and `uk`, with **600+ PII checkpoints** in total. These are the model-backed PII language allow-list. OpenMed also includes validator-backed national-ID coverage for additional ID-only locales such as Polish, Latvian, Slovak, Malay, Filipino, and Danish. diff --git a/clients/go/client.go b/clients/go/client.go index b8681679f..6af2ff339 100644 --- a/clients/go/client.go +++ b/clients/go/client.go @@ -87,6 +87,9 @@ const ( LangTH PIILanguage = "th" LangKO PIILanguage = "ko" LangRO PIILanguage = "ro" + LangUK PIILanguage = "uk" + LangCS PIILanguage = "cs" + LangEL PIILanguage = "el" ) // DeidentificationMethod selects how detected PII spans are transformed by the diff --git a/clients/typescript/src/index.ts b/clients/typescript/src/index.ts index 6f8fb3816..815f9b010 100644 --- a/clients/typescript/src/index.ts +++ b/clients/typescript/src/index.ts @@ -26,7 +26,10 @@ export type PIILanguage = | "id" | "th" | "ko" - | "ro"; + | "ro" + | "uk" + | "cs" + | "el"; export type DeidentificationMethod = | "mask" diff --git a/docs/anonymization.md b/docs/anonymization.md index 6e2824b93..b1ed8d14c 100644 --- a/docs/anonymization.md +++ b/docs/anonymization.md @@ -341,8 +341,8 @@ local attention, sink tokens, RoPE+YaRN, tiktoken `o200k_base`), differing only in their training data: The per-language PII API uses `openmed.core.pii_i18n.SUPPORTED_LANGUAGES` -as its source of truth and supports **17 supported PII language codes**: -`ar`, `de`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, `pt`, `ro`, `te`, `th`, and `tr`. +as its source of truth and supports **20 supported PII language codes**: +`ar`, `cs`, `de`, `el`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, `pt`, `ro`, `te`, `th`, `tr`, and `uk`. These are the model-backed PII language allow-list. Additional validator-backed national-ID providers cover ID-only locales such as Polish, Latvian, Slovak, Malay, Filipino, and Danish without adding diff --git a/docs/api/openapi.json b/docs/api/openapi.json index 91528c45d..f5d018802 100644 --- a/docs/api/openapi.json +++ b/docs/api/openapi.json @@ -196,7 +196,10 @@ "id", "th", "ko", - "ro" + "ro", + "uk", + "cs", + "el" ], "title": "Lang", "type": "string" @@ -417,7 +420,10 @@ "id", "th", "ko", - "ro" + "ro", + "uk", + "cs", + "el" ], "title": "Lang", "type": "string" @@ -540,7 +546,10 @@ "id", "th", "ko", - "ro" + "ro", + "uk", + "cs", + "el" ], "title": "Lang", "type": "string" @@ -636,7 +645,10 @@ "id", "th", "ko", - "ro" + "ro", + "uk", + "cs", + "el" ], "title": "Lang", "type": "string" @@ -752,7 +764,10 @@ "id", "th", "ko", - "ro" + "ro", + "uk", + "cs", + "el" ], "title": "Lang", "type": "string" @@ -876,7 +891,10 @@ "id", "th", "ko", - "ro" + "ro", + "uk", + "cs", + "el" ], "title": "Lang", "type": "string" diff --git a/docs/faq.md b/docs/faq.md index 027f61835..8c5fdae84 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -87,13 +87,13 @@ model argument. Override `model_name` only when you need a specific checkpoint, ### Which languages are supported? -PII extraction and de-identification support **17 supported PII language codes**: -`ar`, `de`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, `pt`, `ro`, `te`, `th`, and `tr`. +PII extraction and de-identification support **20 supported PII language codes**: +`ar`, `cs`, `de`, `el`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, `pt`, `ro`, `te`, `th`, `tr`, and `uk`. These are the model-backed PII language allow-list. Validator-backed national-ID coverage is broader for specific ID-only locales, including Polish, Latvian, Slovak, Malay, Filipino, and Danish. The README keeps a short multilingual example set in -[Multilingual PII](https://github.com/maziyarpanahi/openmed#multilingual-pii-17-model-backed-languages). +[Multilingual PII](https://github.com/maziyarpanahi/openmed#multilingual-pii-20-model-backed-languages). Clinical NER coverage depends on the selected registry model. Check each model's `languages`, `entity_types`, and specialization in the [Model Registry](model-registry.md) before putting it behind an API or batch job. diff --git a/docs/feature-map.md b/docs/feature-map.md index 9d5ee5ee0..11314638a 100644 --- a/docs/feature-map.md +++ b/docs/feature-map.md @@ -15,7 +15,7 @@ coverage, see | Canonical span contracts | Versioned `OpenMedSpan`, schema fingerprints, redaction action schemas, provenance, and compatibility gates. | `openmed/core/schemas/`, [De-identification API](./api/deidentification.md), `examples/v16_policy_audit_release_gates.py` | | Audit and review evidence | Signed audit reports, reproducibility hashes, review bundles, FHIR `Provenance`/`AuditEvent`, audit diffs, and PHI-safe previews. | `openmed/core/audit.py`, `openmed/core/redaction_preview.py`, `openmed/risk/audit_diff.py`, `openmed/clinical/exporters/fhir/provenance.py`, `examples/v16_policy_audit_release_gates.py` | | Runtime de-identification features | `DeidentificationResult.to_dataframe`, surrogate vaults, patient-keyed date shifting, format-preserving redaction, minimum-necessary action selection, streaming redaction, explain traces, section stamping, and risk budgets. | `openmed/core/pii.py`, `openmed/core/surrogate_vault.py`, `openmed/core/date_shift.py`, `openmed/core/anonymizer/format_preserve.py`, `openmed/core/redaction_strength.py`, `openmed/core/streaming.py`, `openmed/core/explain.py`, `openmed/risk/budget.py` | -| Multilingual PII | 17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr in the model-backed allow-list; locale validators, script detection, date/number normalization, deterministic locale PHI generation, and ID-only national-ID providers for additional locales. | `openmed/core/pii_i18n.py`, `openmed/core/script_detect.py`, `openmed/core/locale_formats.py`, `openmed/core/anonymizer/locales.py`, `openmed/training/synthetic/locale_phi.py`, `examples/pii_multilingual_new_languages.py` | +| Multilingual PII | 20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk in the model-backed allow-list; locale validators, script detection, date/number normalization, deterministic locale PHI generation, and ID-only national-ID providers for additional locales. | `openmed/core/pii_i18n.py`, `openmed/core/script_detect.py`, `openmed/core/locale_formats.py`, `openmed/core/anonymizer/locales.py`, `openmed/training/synthetic/locale_phi.py`, `examples/pii_multilingual_new_languages.py` | ## Multimodal And Structured Inputs diff --git a/docs/index.md b/docs/index.md index 8162199f8..c2286f166 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,8 +20,8 @@ coverage, and stronger release evidence: - **Python, Swift, Kotlin/Android, REST, gRPC, React Native, TypeScript, and browser paths** including OpenMedKit, typed REST clients, ONNX/WebGPU, and Transformers.js export bundles. -- **17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, - ko, nl, pt, ro, te, th, and tr** in the model-backed allow-list, with +- **20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, + it, ja, ko, nl, pt, ro, te, th, tr, and uk** in the model-backed allow-list, with locale-aware validation and surrogate generation, plus additional validator-backed national-ID coverage for ID-only locales. - **Release evidence** for leakage heatmaps, model scorecards, threshold diff --git a/docs/languages.md b/docs/languages.md index aa20aa099..f452c9250 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -36,7 +36,9 @@ locale resolution, determinism, and cross-document surrogate vaults, see | Code | Language | Default PII model | Faker locale | Notes | | ------ | ---------- | ---------------------------------------------------------- | ------------ | ----------------------------------------------------------- | | `ar` | Arabic | `OpenMed/OpenMed-PII-Arabic-SnowflakeMed-Large-568M-v1` | `ar_EG` | Egypt is the most-populous Arabic locale; override per call. | +| `cs` | Czech | `OpenMed/privacy-filter-multilingual` | `cs_CZ` | Rodné číslo checksum-aware surrogates. | | `de` | German | `OpenMed/OpenMed-PII-German-SuperClinical-Small-44M-v1` | `de_DE` | Steuer-ID surrogates via `GermanSteuerIdProvider`. | +| `el` | Greek | `OpenMed/privacy-filter-multilingual` | `el_GR` | AMKA Luhn-aware surrogates. | | `en` | English | `OpenMed/OpenMed-PII-SuperClinical-Small-44M-v1` | `en_US` | Default model splits names into `first_name`/`last_name`. | | `es` | Spanish | `OpenMed/OpenMed-PII-Spanish-SuperClinical-Small-44M-v1` | `es_ES` | DNI/NIE checksum-aware surrogates. | | `fr` | French | `OpenMed/OpenMed-PII-French-SuperClinical-Small-44M-v1` | `fr_FR` | NIR / INSEE surrogates via `fr_FR.ssn`. | @@ -52,6 +54,7 @@ locale resolution, determinism, and cross-document surrogate vaults, see | `te` | Telugu | `OpenMed/OpenMed-PII-Telugu-SuperClinical-Large-434M-v1` | `en_IN` | No Faker Telugu locale — `en_IN` approximation (warns once). | | `th` | Thai | `OpenMed/privacy-filter-multilingual` | `th_TH` | Served by the multilingual privacy filter; Thai NID-aware. | | `tr` | Turkish | `OpenMed/OpenMed-PII-Turkish-SuperClinical-Small-44M-v1` | `tr_TR` | TCKN surrogates. | +| `uk` | Ukrainian | `OpenMed/privacy-filter-multilingual` | `uk_UA` | RNOKPP checksum-aware surrogates. | Codes outside this list (for example `zh`, `pl`, `lv`, `sk`, `ms`, `tl`, `da`) are **not** model-backed PII languages. Several of them still have @@ -76,6 +79,15 @@ Before: المريضة ليلى حسن، الهاتف +20 10 1234 5678 After: المريضة [NAME]، الهاتف [PHONE] ``` +### Czech — `cs` + +- Model: `OpenMed/privacy-filter-multilingual` · locale `cs_CZ` + +```text +Before: Pacient Jan Novák, rodné číslo 751116/0008 +After: Pacient [NAME], rodné číslo [ID] +``` + ### German — `de` - Model: `OpenMed/OpenMed-PII-German-SuperClinical-Small-44M-v1` · locale `de_DE` @@ -85,6 +97,15 @@ Before: Patientin Anna Müller, Steuer-ID 86095742719 After: Patientin [NAME], Steuer-ID [ID] ``` +### Greek — `el` + +- Model: `OpenMed/privacy-filter-multilingual` · locale `el_GR` + +```text +Before: Ασθενής Γιώργος Παπαδόπουλος, ΑΜΚΑ 16117508024 +After: Ασθενής [NAME], ΑΜΚΑ [ID] +``` + ### English — `en` - Model: `OpenMed/OpenMed-PII-SuperClinical-Small-44M-v1` · locale `en_US` @@ -221,3 +242,12 @@ After: ผู้ป่วย [NAME] โทร [PHONE] Before: Hasta Ayşe Yılmaz, TCKN 10000000146 After: Hasta [NAME], TCKN [ID] ``` + +### Ukrainian — `uk` + +- Model: `OpenMed/privacy-filter-multilingual` · locale `uk_UA` + +```text +Before: Пацієнтка Олена Коваль, РНОКПП 2974281300 +After: Пацієнтка [NAME], РНОКПП [ID] +``` diff --git a/docs/rest-recipes.md b/docs/rest-recipes.md index ca455ebac..f2f9884fc 100644 --- a/docs/rest-recipes.md +++ b/docs/rest-recipes.md @@ -174,9 +174,10 @@ hardware): ## Extract PII — `POST /pii/extract` Detect personally identifiable information. Unless `model_name` is set, OpenMed -selects the recommended PII model for `lang`. The 17 supported PII language -codes: `ar`, `de`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, `ko`, `nl`, -`pt`, `ro`, `te`, `th`, and `tr`. `confidence_threshold` defaults to `0.5`. +selects the recommended PII model for `lang`. The 20 supported PII language +codes: `ar`, `cs`, `de`, `el`, `en`, `es`, `fr`, `he`, `hi`, `id`, `it`, `ja`, +`ko`, `nl`, `pt`, `ro`, `te`, `th`, `tr`, and `uk`. `confidence_threshold` +defaults to `0.5`. ```bash curl -sS --max-time 310 -X POST "$OPENMED_URL/pii/extract" \ diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 24fffa168..85a094154 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -383,12 +383,12 @@ text = validate_input(user_supplied_text, max_length=2000, allow_empty=False) **Symptom.** A PII call with an unrecognized `lang` raises: ```text -ValueError: Unsupported language 'xx'. Supported: ['ar', 'de', 'en', 'es', 'fr', 'he', 'hi', 'id', 'it', 'ja', 'ko', 'nl', 'pt', 'ro', 'te', 'th', 'tr'] +ValueError: Unsupported language 'xx'. Supported: ['ar', 'cs', 'de', 'el', 'en', 'es', 'fr', 'he', 'hi', 'id', 'it', 'ja', 'ko', 'nl', 'pt', 'ro', 'te', 'th', 'tr', 'uk'] ``` -**Cause.** PII extraction and de-identification support **17 supported PII language codes: ar, de, en, es, -fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr**. Passing anything outside that set (or a mistyped -code) raises this error. +**Cause.** PII extraction and de-identification support **20 supported PII language codes: ar, cs, de, el, +en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk**. Passing anything outside that set (or a +mistyped code) raises this error. **Fix.** Use one of the supported codes with `extract_pii(..., lang="")`. Clinical NER coverage depends on the selected registry model — check each model's `languages` in the diff --git a/docs/website/assets/script.js b/docs/website/assets/script.js index 01a9fc78b..6b7b1c66c 100644 --- a/docs/website/assets/script.js +++ b/docs/website/assets/script.js @@ -117,7 +117,7 @@ function moonSVG() { return ``; } -/* ---------- PHI Demo (cycles through 7 of the 17 languages OpenMed supports) ------ */ +/* ---------- PHI Demo (cycles through 7 of the 20 languages OpenMed supports) ------ */ const PHI_SAMPLES = [ { lang: "en", diff --git a/docs/website/index.html b/docs/website/index.html index 87215707f..05018c26e 100644 --- a/docs/website/index.html +++ b/docs/website/index.html @@ -75,13 +75,13 @@ "@type": "FAQPage", "mainEntity": [ { "@type": "Question", "name": "What is OpenMed, exactly?", - "acceptedAnswer": { "@type": "Answer", "text": "An open-source medical LLM toolkit. Specialized transformer models fine-tuned for biomedical named-entity recognition: diseases, drugs, genes, anatomy, chemicals, oncology, plus PII extraction and de-identification across 17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr. These are the model-backed PII language allow-list. Ships as a Python package, Dockerized FastAPI and gRPC services, Swift and Kotlin OpenMedKit packages, React Native bridge, and browser runtime. Apache-2.0, no vendor lock-in, runs on your infrastructure." } }, + "acceptedAnswer": { "@type": "Answer", "text": "An open-source medical LLM toolkit. Specialized transformer models fine-tuned for biomedical named-entity recognition: diseases, drugs, genes, anatomy, chemicals, oncology, plus PII extraction and de-identification across 20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk. These are the model-backed PII language allow-list. Ships as a Python package, Dockerized FastAPI and gRPC services, Swift and Kotlin OpenMedKit packages, React Native bridge, and browser runtime. Apache-2.0, no vendor lock-in, runs on your infrastructure." } }, { "@type": "Question", "name": "Are these generative LLMs or something else?", "acceptedAnswer": { "@type": "Answer", "text": "Encoder transformers (BERT, ELECTRA, DeBERTa families), not generative chat models. They do extraction and classification, pulling structured entities out of unstructured clinical text, and stay small enough to run on a laptop or a phone. The reference paper reports new state-of-the-art on 10 of 12 biomedical NER benchmarks. Complementary to the larger generative medical LLM category, not a replacement." } }, { "@type": "Question", "name": "Where does my data go when I use OpenMed?", "acceptedAnswer": { "@type": "Answer", "text": "Nowhere you don't send it. You download the models once (Hugging Face or a private mirror) and inference runs wherever you run the Python process, the Docker container, or the Swift app: your laptop, your VPC, an on-prem server, or air-gapped hardware. No telemetry, no license check-in, no outbound calls at runtime. PHI stays on your side of the fence by default." } }, { "@type": "Question", "name": "Does OpenMed support HIPAA-aligned workflows?", - "acceptedAnswer": { "@type": "Answer", "text": "The PII catalog covers all 18 HIPAA Safe Harbor identifiers across 17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr, the model-backed PII language allow-list, with locale-aware validators for SSN, NIR, Steuer-ID, Codice Fiscale, DNI, BSN, CPF/CNPJ, RRN, CNP, Luhn checks, and additional ID-only national-ID validators. Models are trained on de-identified, ethically sourced corpora. OpenMed provides the technical controls; the legal compliance boundary lives in your deployment." } }, + "acceptedAnswer": { "@type": "Answer", "text": "The PII catalog covers all 18 HIPAA Safe Harbor identifiers across 20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk, the model-backed PII language allow-list, with locale-aware validators for SSN, NIR, Steuer-ID, Codice Fiscale, DNI, BSN, CPF/CNPJ, RRN, CNP, Luhn checks, and additional ID-only national-ID validators. Models are trained on de-identified, ethically sourced corpora. OpenMed provides the technical controls; the legal compliance boundary lives in your deployment." } }, { "@type": "Question", "name": "Can I fine-tune OpenMed models for my own vocabulary?", "acceptedAnswer": { "@type": "Answer", "text": "Yes. Models are published on Hugging Face under permissive licensing with full training recipes. The reference approach combines lightweight domain-adaptive pre-training on a 350k-passage biomedical corpus with parameter-efficient LoRA fine-tuning, updating less than 1.5% of model parameters and completing in under 12 hours on a single GPU (<1.2 kg CO2e). Tokenizer assets and starter notebooks are in the openmed-starter repo." } }, { "@type": "Question", "name": "Do I need OpenMed Agent to use OpenMed?", @@ -240,7 +240,7 @@

- Live PHI detection · 17 model-backed languages + Live PHI detection · 20 model-backed languages ▶ watch
@@ -278,7 +278,7 @@

17
-
Model-backed PII languages: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr
+
Model-backed PII languages: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, uk
<1.2kg
@@ -326,7 +326,7 @@

Four lines to production
extract_pii / deidentify -
Language-aware across 17 model-backed languages
+
Language-aware across 20 model-backed languages
  • @@ -455,7 +455,7 @@

    Clinical text de-identification, built for HIPAA & GDPR.

    - Language-aware PII models across 17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr, plus validator-backed national-ID coverage for additional ID-only locales. These are the model-backed PII language allow-list. Process data locally, so your PHI never leaves your environment. + Language-aware PII models across 20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk, plus validator-backed national-ID coverage for additional ID-only locales. These are the model-backed PII language allow-list. Process data locally, so your PHI never leaves your environment.

    • @@ -545,11 +545,11 @@

    Flexible redaction methods

    HIPAA Safe Harbor ready

    -

    Detects all 18 HIPAA Safe Harbor identifiers, part of a wider 55+ PII entity catalog, across 17 model-backed PII language codes, with additional validator-backed national-ID coverage for ID-only locales.

    +

    Detects all 18 HIPAA Safe Harbor identifiers, part of a wider 55+ PII entity catalog, across 20 model-backed PII language codes, with additional validator-backed national-ID coverage for ID-only locales.

    18 Safe Harbor 55+ entity types - 17 model-backed languages + 20 model-backed languages
    @@ -866,7 +866,7 @@

    Questions from the clinical, ML, and compliance teams.+
    -

    An open-source medical NLP toolkit. Specialized transformer models fine-tuned for biomedical named-entity recognition: diseases, drugs, genes, anatomy, chemicals, oncology, plus PII extraction and de-identification across 17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr. These are the model-backed PII language allow-list. Ships as a Python package, Dockerized FastAPI and gRPC services, Swift and Kotlin OpenMedKit packages, React Native bridge, and browser runtime. Apache-2.0, no vendor lock-in, runs on your infrastructure.

    +

    An open-source medical NLP toolkit. Specialized transformer models fine-tuned for biomedical named-entity recognition: diseases, drugs, genes, anatomy, chemicals, oncology, plus PII extraction and de-identification across 20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk. These are the model-backed PII language allow-list. Ships as a Python package, Dockerized FastAPI and gRPC services, Swift and Kotlin OpenMedKit packages, React Native bridge, and browser runtime. Apache-2.0, no vendor lock-in, runs on your infrastructure.

    @@ -893,7 +893,7 @@

    Questions from the clinical, ML, and compliance teams.+

    -

    The PII catalog covers all 18 HIPAA Safe Harbor identifiers across 17 supported PII language codes: ar, de, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, and tr, the model-backed PII language allow-list, with locale-aware validators for SSN, NIR, Steuer-ID, Codice Fiscale, DNI, BSN, CPF/CNPJ, RRN, CNP, Luhn checks, and additional ID-only national-ID validators. Models are trained on de-identified, ethically sourced corpora. OpenMed provides the technical controls (on-device processing, configurable thresholds, multiple redaction methods); the legal compliance boundary lives in your deployment.

    +

    The PII catalog covers all 18 HIPAA Safe Harbor identifiers across 20 supported PII language codes: ar, cs, de, el, en, es, fr, he, hi, id, it, ja, ko, nl, pt, ro, te, th, tr, and uk, the model-backed PII language allow-list, with locale-aware validators for SSN, NIR, Steuer-ID, Codice Fiscale, DNI, BSN, CPF/CNPJ, RRN, CNP, Luhn checks, and additional ID-only national-ID validators. Models are trained on de-identified, ethically sourced corpora. OpenMed provides the technical controls (on-device processing, configurable thresholds, multiple redaction methods); the legal compliance boundary lives in your deployment.

    diff --git a/models.jsonl b/models.jsonl index 916b675b5..1a91c4bf2 100644 --- a/models.jsonl +++ b/models.jsonl @@ -1501,16 +1501,16 @@ {"repo_id":"OpenMed/privacy-filter-ai4privacy-multilingual","family":"PII","task":"token-classification","languages":["de","en","es","fr","it","nl"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-400k","micro_f1":null,"recall":null},"arxiv":null,"license":null,"reproducibility_hash":"sha256:d19f0d8d99f6110fb628c165f2abd74d5ad1548cade42e1b23ff08fccbc92fcc","released":"2026-04-27"} {"repo_id":"OpenMed/privacy-filter-mlx","family":"PII","task":"token-classification","languages":["en"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["mlx-fp","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":null,"micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:c5cc91ec36a04a237164f1b540736f81cb45c4ce47b987c399df5d2a2e152458","released":"2026-04-23"} {"repo_id":"OpenMed/privacy-filter-mlx-8bit","family":"PII","task":"token-classification","languages":["en"],"tier":null,"param_count":8000000000,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["mlx-8bit","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":null,"micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:fdea6d0c7d53b939197e35f2bacab8aeb29c7977efe0e91a11957ec1f2c7125d","released":"2026-04-24"} -{"repo_id":"OpenMed/privacy-filter-multilingual","family":"PII","task":"token-classification","languages":["ar","de","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:0361a60a757838d1705d7f4eb964d06ce62a3444ce857a50f04475a634ad148c","released":"2026-05-03"} -{"repo_id":"OpenMed/privacy-filter-multilingual-mlx","family":"PII","task":"token-classification","languages":["ar","de","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr"],"tier":null,"param_count":null,"architecture":"privacy-filter-multilingual","base_model":"OpenMed/privacy-filter-multilingual","formats":["mlx-fp","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:ee10aab29e25b15585c19b90e6f282d5b91264856e8b43655969ecc2a20adefa","released":"2026-05-04"} -{"repo_id":"OpenMed/privacy-filter-multilingual-mlx-8bit","family":"PII","task":"token-classification","languages":["ar","de","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr"],"tier":null,"param_count":8000000000,"architecture":"privacy-filter-multilingual","base_model":"OpenMed/privacy-filter-multilingual","formats":["mlx-8bit","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:9b8ada55fc4ecdee7b11020fc571de344db3d184a70f852859a056e9268d490a","released":"2026-05-04"} -{"repo_id":"OpenMed/privacy-filter-multilingual-v2","family":"PII","task":"token-classification","languages":["ar","de","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:e599d20337f10da7dc09602da6befb361bea6dcc9ddbab80729235e6d73a02ca","released":"2026-05-06"} +{"repo_id":"OpenMed/privacy-filter-multilingual","family":"PII","task":"token-classification","languages":["ar","cs","de","el","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr","uk"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:0361a60a757838d1705d7f4eb964d06ce62a3444ce857a50f04475a634ad148c","released":"2026-05-03"} +{"repo_id":"OpenMed/privacy-filter-multilingual-mlx","family":"PII","task":"token-classification","languages":["ar","cs","de","el","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr","uk"],"tier":null,"param_count":null,"architecture":"privacy-filter-multilingual","base_model":"OpenMed/privacy-filter-multilingual","formats":["mlx-fp","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:ee10aab29e25b15585c19b90e6f282d5b91264856e8b43655969ecc2a20adefa","released":"2026-05-04"} +{"repo_id":"OpenMed/privacy-filter-multilingual-mlx-8bit","family":"PII","task":"token-classification","languages":["ar","cs","de","el","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr","uk"],"tier":null,"param_count":8000000000,"architecture":"privacy-filter-multilingual","base_model":"OpenMed/privacy-filter-multilingual","formats":["mlx-8bit","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:9b8ada55fc4ecdee7b11020fc571de344db3d184a70f852859a056e9268d490a","released":"2026-05-04"} +{"repo_id":"OpenMed/privacy-filter-multilingual-v2","family":"PII","task":"token-classification","languages":["ar","cs","de","el","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr","uk"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:e599d20337f10da7dc09602da6befb361bea6dcc9ddbab80729235e6d73a02ca","released":"2026-05-06"} {"repo_id":"OpenMed/privacy-filter-nemotron","family":"PII","task":"token-classification","languages":["en"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"nvidia/Nemotron-PII","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:4e48757f0c46473fd197349b0e0bd7e1e91fed94874779fa0fee42ce616ef539","released":"2026-04-28"} {"repo_id":"OpenMed/privacy-filter-nemotron-mlx","family":"PII","task":"token-classification","languages":["en"],"tier":null,"param_count":null,"architecture":"privacy-filter-nemotron","base_model":"OpenMed/privacy-filter-nemotron","formats":["mlx-fp","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"nvidia/Nemotron-PII","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:8d066cc5318174155880d36163dae87945c294b246509b5fd0ae53488cdff432","released":"2026-04-28"} {"repo_id":"OpenMed/privacy-filter-nemotron-mlx-8bit","family":"PII","task":"token-classification","languages":["en"],"tier":null,"param_count":8000000000,"architecture":"privacy-filter-nemotron","base_model":"OpenMed/privacy-filter-nemotron","formats":["mlx-8bit","pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"nvidia/Nemotron-PII","micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:03ff66c2778b8e9753b5cd9107d156a7d911b7f51c47c576cf63b7d36f860d95","released":"2026-04-28"} {"repo_id":"OpenMed/privacy-filter-nemotron-piimb-v2","family":"PII","task":"token-classification","languages":["de","en","es","fr","it","nl","pt"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"nvidia/Nemotron-PII","micro_f1":null,"recall":null},"arxiv":null,"license":"other","reproducibility_hash":"sha256:c75c1f0b1c892524106014a77734387a3b059d1d9dbad8a122c8774a63c97641","released":"2026-05-03"} {"repo_id":"OpenMed/privacy-filter-nemotron-v2","family":"PII","task":"token-classification","languages":["de","en","es","fr","it","nl","pt"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"nvidia/Nemotron-PII","micro_f1":null,"recall":null},"arxiv":null,"license":"other","reproducibility_hash":"sha256:307b7a1b443ecb95b0430c6ece59ff9c65c53dbd453bb6b2fe23d0e109630fbe","released":"2026-05-03"} -{"repo_id":"OpenMed/privacy-filter-openmed-pii54-multilingual","family":"PII","task":"token-classification","languages":["ar","de","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr"],"tier":null,"param_count":null,"architecture":null,"base_model":null,"formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"other","reproducibility_hash":"sha256:9a7aed8b87f32bca8d00c19e2972e29b46e73b1497e24da1242e035fcd9a4b70","released":"2026-04-28"} +{"repo_id":"OpenMed/privacy-filter-openmed-pii54-multilingual","family":"PII","task":"token-classification","languages":["ar","cs","de","el","en","es","fr","he","hi","id","it","ja","nl","pt","ro","te","th","tr","uk"],"tier":null,"param_count":null,"architecture":null,"base_model":null,"formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"ai4privacy/pii-masking-200k","micro_f1":null,"recall":null},"arxiv":null,"license":"other","reproducibility_hash":"sha256:9a7aed8b87f32bca8d00c19e2972e29b46e73b1497e24da1242e035fcd9a4b70","released":"2026-04-28"} {"repo_id":"OpenMed/privacy-filter-piimb-fine-grained","family":"PII","task":"token-classification","languages":["de","en","es","fr","it","nl","pt"],"tier":null,"param_count":null,"architecture":"privacy-filter","base_model":"privacy-filter","formats":["pytorch"],"canonical_labels":["PERSON","FIRST_NAME","LAST_NAME","MIDDLE_NAME","PREFIX","USERNAME","EMAIL","PHONE","URL","LOCATION","STREET_ADDRESS","BUILDING_NUMBER","ZIPCODE","GPS_COORDINATES","ORDINAL_DIRECTION","DATE","DATE_OF_BIRTH","TIME","AGE","ID_NUM","SSN","ACCOUNT_NUMBER","PASSWORD","PIN","API_KEY","CREDIT_CARD","CREDIT_CARD_ISSUER","CVV","IBAN","BIC","AMOUNT","CURRENCY","BITCOIN_ADDRESS","ETHEREUM_ADDRESS","LITECOIN_ADDRESS","MASKED_NUMBER","GENDER","EYE_COLOR","HEIGHT","ORGANIZATION","JOB_TITLE","JOB_DEPARTMENT","OCCUPATION","IP_ADDRESS","MAC_ADDRESS","USER_AGENT","VIN","VEHICLE_REGISTRATION","IMEI","OTHER"],"benchmark":{"dataset":"nvidia/Nemotron-PII","micro_f1":null,"recall":null},"arxiv":null,"license":"other","reproducibility_hash":"sha256:a45a1c2f956686407ac8846aec2815796670e8b6104f02ad2a2a9860b2c043c3","released":"2026-04-29"} {"repo_id":"OpenMed/Qwen2.5-3B-MedVL","family":"Vision","task":"visual-question-answering","languages":[],"tier":null,"param_count":3000000000,"architecture":"qwen","base_model":"Qwen/Qwen2.5-VL-3B-Instruct","formats":["pytorch"],"canonical_labels":[],"benchmark":{"dataset":null,"micro_f1":null,"recall":null},"arxiv":null,"license":"apache-2.0","reproducibility_hash":"sha256:34185498a5dcacc59a1fd9b0262017fe32c061e96b1608c1ed80c7d76bc1d74d","released":"2026-03-23"} {"repo_id":"OpenMed/Qwen2.5-VL-3B-Medical-v1","family":"General","task":"unknown","languages":[],"tier":null,"param_count":3000000000,"architecture":"qwen","base_model":null,"formats":["pytorch"],"canonical_labels":[],"benchmark":{"dataset":null,"micro_f1":null,"recall":null},"arxiv":null,"license":null,"reproducibility_hash":"sha256:839b80b14bae3a584808c25c87455b90221467637a9b9ebc93012ad3ed2b1d48","released":"2026-03-03"} diff --git a/openmed/core/anonymizer/engine.py b/openmed/core/anonymizer/engine.py index ea333e525..e532a00fc 100644 --- a/openmed/core/anonymizer/engine.py +++ b/openmed/core/anonymizer/engine.py @@ -321,7 +321,9 @@ def format_preserving_surrogate( "en_IN", "pt_PT", "pt_BR", + "uk_UA", "cs_CZ", + "el_GR", } ) diff --git a/openmed/core/anonymizer/locales.py b/openmed/core/anonymizer/locales.py index 3f2066df4..0966cb7c7 100644 --- a/openmed/core/anonymizer/locales.py +++ b/openmed/core/anonymizer/locales.py @@ -54,6 +54,7 @@ "pl": "pl_PL", "lv": "lv_LV", "ko": "ko_KR", + "uk": "uk_UA", "cs": "cs_CZ", "sk": "sk_SK", "ms": "ms_MY", @@ -115,6 +116,7 @@ "pl": ("pl_PL", "pesel"), # PESEL "lv": ("lv_LV", "personas_kods"), "ko": ("ko_KR", "korean_rrn"), # RRN + "uk": ("uk_UA", "rnokpp"), # Ukrainian taxpayer registration number "cs": ("cs_CZ", "rodne_cislo"), # Czech rodne cislo (shared provider) "sk": ("sk_SK", "rodne_cislo"), # Slovak rodne cislo "ms": ("ms_MY", "mykad"), # Malaysian MyKad / NRIC diff --git a/openmed/core/anonymizer/providers/__init__.py b/openmed/core/anonymizer/providers/__init__.py index 99945d0e7..0764d197f 100644 --- a/openmed/core/anonymizer/providers/__init__.py +++ b/openmed/core/anonymizer/providers/__init__.py @@ -26,6 +26,7 @@ RomanianCNPProvider, SerbianJmbgProvider, SpanishDNIProvider, + UkrainianRnokppProvider, VietnameseIdProvider, generate_bic, generate_bulgarian_egn, @@ -43,6 +44,7 @@ generate_portuguese_nif, generate_romanian_cnp, generate_teudat_zehut, + generate_ukrainian_rnokpp, generate_vietnamese_cccd, generate_vietnamese_cmnd, register_clinical_providers, @@ -76,6 +78,7 @@ "RomanianCNPProvider", "SerbianJmbgProvider", "SpanishDNIProvider", + "UkrainianRnokppProvider", "VietnameseIdProvider", "generate_bic", "generate_bulgarian_egn", @@ -86,6 +89,7 @@ "generate_indonesian_nik", "generate_jmbg", "generate_teudat_zehut", + "generate_ukrainian_rnokpp", "generate_vietnamese_cccd", "generate_vietnamese_cmnd", "generate_korean_rrn", diff --git a/openmed/core/anonymizer/providers/clinical_ids.py b/openmed/core/anonymizer/providers/clinical_ids.py index e46b75dc4..ed06ecaee 100644 --- a/openmed/core/anonymizer/providers/clinical_ids.py +++ b/openmed/core/anonymizer/providers/clinical_ids.py @@ -27,6 +27,7 @@ - Philippine PhilSys PSN and PhilHealth PIN structural formats - Danish CPR / personnummer with a decodable embedded birth date - Thai national ID (13 digits with a weighted mod-11 checksum) + - Ukrainian RNOKPP (10 digits with a weighted mod-11 checksum) - Polish PESEL, Latvian personas kods, South Korean RRN, and Slovak rodne cislo - UK NHS Number, a patient health identifier validated with the NHS @@ -1447,6 +1448,30 @@ def rodne_cislo(self) -> str: return generate_rodne_cislo(rng=self.generator.random) +# --------------------------------------------------------------------------- +# Ukrainian RNOKPP (10 digits, weighted modulo-11 checksum) +# --------------------------------------------------------------------------- + + +def generate_ukrainian_rnokpp(*, rng: random.Random | None = None) -> str: + """Generate a checksum-valid Ukrainian RNOKPP surrogate.""" + source = rng or random.Random() + day_count = source.randint(1, 50000) + serial = source.randint(0, 999) + gender_digit = source.randint(0, 9) + body = f"{day_count:05d}{serial:03d}{gender_digit}" + weights = (-1, 5, 7, 9, 4, 6, 10, 5, 7) + check = sum(int(digit) * weight for digit, weight in zip(body, weights)) % 11 % 10 + return body + str(check) + + +class UkrainianRnokppProvider(BaseProvider): + """Generates valid Ukrainian RNOKPP taxpayer registration numbers.""" + + def rnokpp(self) -> str: + return generate_ukrainian_rnokpp(rng=self.generator.random) + + # --------------------------------------------------------------------------- # Thai national ID (13 digits, weighted mod-11 checksum) # --------------------------------------------------------------------------- @@ -1948,6 +1973,7 @@ def register_clinical_providers(faker) -> None: "SpanishNIEProvider", "UKNHSNumberProvider", "UKNINOProvider", + "UkrainianRnokppProvider", "generate_australian_medicare", "generate_australian_tfn", "generate_bc_phn", @@ -1979,6 +2005,7 @@ def register_clinical_providers(faker) -> None: "generate_vietnamese_cccd", "generate_vietnamese_cmnd", "generate_uk_nhs_number", + "generate_ukrainian_rnokpp", "generate_unified_social_credit_code", "id_subtype_for_entity_type", "register_clinical_providers", diff --git a/openmed/core/anonymizer/providers/registry_ids.py b/openmed/core/anonymizer/providers/registry_ids.py index 99a9537ec..57e6ec5fe 100644 --- a/openmed/core/anonymizer/providers/registry_ids.py +++ b/openmed/core/anonymizer/providers/registry_ids.py @@ -61,6 +61,7 @@ validate_turkish_tckn, validate_uk_nhs_number, validate_uk_nino, + validate_ukrainian_rnokpp, validate_vietnamese_cccd, validate_vietnamese_cmnd, ) @@ -94,6 +95,7 @@ ThaiNationalIdProvider, UKNHSNumberProvider, UKNINOProvider, + UkrainianRnokppProvider, VietnameseIdProvider, validate_australian_medicare, validate_australian_tfn, @@ -381,6 +383,13 @@ def _register_builtin_specs() -> None: faker_method="isikukood", faker_provider=EstonianIsikukoodProvider, ) + _register_aliases( + ("uk", "uk_UA"), + id_type="rnokpp", + validate=validate_ukrainian_rnokpp, + faker_method="rnokpp", + faker_provider=UkrainianRnokppProvider, + ) _register_aliases( ("el", "el_GR"), id_type="amka", diff --git a/openmed/core/anonymizer/registry.py b/openmed/core/anonymizer/registry.py index 251cd1ab6..b13d35489 100644 --- a/openmed/core/anonymizer/registry.py +++ b/openmed/core/anonymizer/registry.py @@ -129,8 +129,10 @@ def _gen_gps(faker, original, *, locale): "fil_PH", "da_DK", "th_TH", + "uk_UA", "cs_CZ", "sk_SK", + "el_GR", } ) @@ -182,6 +184,7 @@ def _gen_age(faker, original, *, locale): "lv_LV": "personas_kods", "ko_KR": "korean_rrn", "th_TH": "thai_national_id", + "uk_UA": "rnokpp", "sk_SK": "rodne_cislo", "cs_CZ": "rodne_cislo", "ro_RO": "romanian_cnp", diff --git a/openmed/core/model_card.py b/openmed/core/model_card.py index 2a08baea8..1794a43e1 100644 --- a/openmed/core/model_card.py +++ b/openmed/core/model_card.py @@ -406,7 +406,7 @@ def _render_android_onnx_model_card( "clinical AI stack:", "", "- **2,000+ medical models** for clinical NER, biomedical extraction, and privacy.", - "- **PII detection and de-identification** across 55+ identifier types and 17 languages.", + "- **PII detection and de-identification** across 55+ identifier types and 20 languages.", "- **Python, MLX, Swift, Android, React Native, Web, REST, and gRPC** runtimes.", "- **Structured and multimodal intake** for OCR, documents, DICOM, FHIR, and HL7.", "- **Offline and air-gapped deployment** with no telemetry by default.", diff --git a/openmed/core/pii_i18n.py b/openmed/core/pii_i18n.py index 5f6e29925..68f62fbd1 100644 --- a/openmed/core/pii_i18n.py +++ b/openmed/core/pii_i18n.py @@ -53,6 +53,9 @@ "th", "ko", "ro", + "uk", + "cs", + "el", } # Languages with validator-backed national-ID coverage but no bundled default @@ -70,8 +73,6 @@ "hr", "bg", "fi", - "cs", - "el", "vi", } @@ -93,6 +94,9 @@ "th": "Thai", "ko": "Korean", "ro": "Romanian", + "uk": "Ukrainian", + "cs": "Czech", + "el": "Greek", } LANGUAGE_MODEL_PREFIX: Dict[str, str] = { @@ -113,6 +117,9 @@ "th": "Thai-", "ko": "Korean-", "ro": "Romanian-", + "uk": "Ukrainian-", + "cs": "Czech-", + "el": "Greek-", } DEFAULT_PII_MODELS: Dict[str, str] = { @@ -133,6 +140,9 @@ "th": "OpenMed/privacy-filter-multilingual", "ko": "OpenMed/OpenMed-PII-Korean-NomicMed-Large-395M-v1", "ro": "OpenMed/privacy-filter-multilingual", + "uk": "OpenMed/privacy-filter-multilingual", + "cs": "OpenMed/privacy-filter-multilingual", + "el": "OpenMed/privacy-filter-multilingual", } @@ -1032,6 +1042,34 @@ def _latvian_personas_kods_check_digit(digits: list[int]) -> int: ) +def validate_ukrainian_rnokpp(text: str) -> bool: + """Validate a Ukrainian RNOKPP taxpayer registration number. + + RNOKPP values contain ten digits. The last digit is the modulo-11, + modulo-10 checksum of the first nine digits weighted by + ``(-1, 5, 7, 9, 4, 6, 10, 5, 7)``. + + Args: + text: Candidate ten-digit RNOKPP value. + + Returns: + ``True`` when the value has the required shape and checksum. + """ + if not isinstance(text, str): + return False + + digits = text.strip() + if re.fullmatch(r"[0-9]{10}", digits) is None: + return False + + weights = (-1, 5, 7, 9, 4, 6, 10, 5, 7) + numbers = [int(digit) for digit in digits] + check_digit = ( + sum(weight * digit for weight, digit in zip(weights, numbers[:9])) % 11 % 10 + ) + return numbers[9] == check_digit + + def validate_greek_amka(text: str) -> bool: """Validate a Greek AMKA social-security number. @@ -1814,6 +1852,48 @@ def validate_romanian_cnp(text: str) -> bool: "noiembrie", "decembrie", ], + "uk": [ + "січень", + "лютий", + "березень", + "квітень", + "травень", + "червень", + "липень", + "серпень", + "вересень", + "жовтень", + "листопад", + "грудень", + ], + "cs": [ + "leden", + "únor", + "březen", + "duben", + "květen", + "červen", + "červenec", + "srpen", + "září", + "říjen", + "listopad", + "prosinec", + ], + "el": [ + "Ιανουάριος", + "Φεβρουάριος", + "Μάρτιος", + "Απρίλιος", + "Μάιος", + "Ιούνιος", + "Ιούλιος", + "Αύγουστος", + "Σεπτέμβριος", + "Οκτώβριος", + "Νοέμβριος", + "Δεκέμβριος", + ], } @@ -4884,6 +4964,102 @@ def validate_unified_social_credit_code(text: str) -> bool: ), ] +# --------------------------------------------------------------------------- +# Ukrainian PII patterns (Cyrillic script) +# --------------------------------------------------------------------------- + +_UKRAINIAN_MONTH_PATTERN = ( + r"січня|січень|лютого|лютий|березня|березень|квітня|квітень|" + r"травня|травень|червня|червень|липня|липень|серпня|серпень|" + r"вересня|вересень|жовтня|жовтень|листопада|листопад|грудня|грудень" +) + +_UKRAINIAN_STREET_MARKER = ( + r"вулиця|вул\.|проспект|просп\.|провулок|пров\.|бульвар|бул\.|площа|шосе" +) + +_UKRAINIAN_PII_PATTERNS: List[PIIPattern] = [ + PIIPattern( + r"\b\d{1,2}[./-]\d{1,2}[./-]\d{2,4}\b", + "date", + priority=9, + base_score=0.6, + context_words=[ + "дата", + "народження", + "народився", + "народилася", + "прийом", + "виписка", + ], + context_boost=0.3, + ), + PIIPattern( + rf"\b\d{{1,2}}\s+(?:{_UKRAINIAN_MONTH_PATTERN})\s+\d{{4}}\b", + "date", + priority=8, + base_score=0.7, + context_words=[ + "дата", + "народження", + "народився", + "народилася", + "прийом", + "виписка", + ], + context_boost=0.25, + flags=re.IGNORECASE, + ), + PIIPattern( + r"(? bool: "sr": _SERBIAN_PII_PATTERNS, "hu": _HUNGARIAN_PII_PATTERNS, "et": _ESTONIAN_PII_PATTERNS, + "uk": _UKRAINIAN_PII_PATTERNS, "el": _GREEK_PII_PATTERNS, "cs": _CZECH_PII_PATTERNS, "vi": _VIETNAMESE_PII_PATTERNS, @@ -5746,6 +5923,21 @@ def validate_unified_social_credit_code(text: str) -> bool: "LOCATION": ["Tallinn", "Tartu", "Parnu"], "ZIPCODE": ["10115", "50090", "80010"], }, + "uk": { + "NAME": ["Олена Коваль", "Андрій Шевченко", "Ірина Бондар", "Тарас Мельник"], + "FIRST_NAME": ["Олена", "Андрій", "Ірина", "Тарас"], + "LAST_NAME": ["Коваль", "Шевченко", "Бондар", "Мельник"], + "EMAIL": ["patsiient@example.ua", "kontakt@example.org"], + "PHONE": ["+380 67 123 45 67", "044 123 45 67"], + "ID_NUM": ["2974281300", "3695007088"], + "STREET_ADDRESS": ["вулиця Хрещатик 22", "проспект Свободи 15"], + "URL_PERSONAL": ["https://example.ua"], + "USERNAME": ["patsiient123", "korystuvach456"], + "DATE": ["16.11.1975", "16 листопада 1975"], + "AGE": ["45", "62", "38"], + "LOCATION": ["Київ", "Львів", "Одеса"], + "ZIPCODE": ["01001", "79000", "65000"], + }, "cs": { "NAME": ["Jana Nováková", "Petr Novák", "Marie Svobodová", "Tomáš Král"], "FIRST_NAME": ["Jana", "Petr", "Marie", "Tomáš"], diff --git a/openmed/core/script_detect.py b/openmed/core/script_detect.py index 0ca90d682..12d40bcea 100644 --- a/openmed/core/script_detect.py +++ b/openmed/core/script_detect.py @@ -29,15 +29,15 @@ ) SCRIPT_LANGUAGE_HINTS: dict[str, tuple[str, ...]] = { - "Latin": ("en", "fr", "de", "it", "es", "nl", "pt", "tr"), + "Latin": ("en", "fr", "de", "it", "es", "nl", "pt", "tr", "cs"), "Arabic": ("ar",), "Han": ("ja",), "Hiragana/Katakana": ("ja",), "Hangul": ("ko",), - "Cyrillic": ("en",), + "Cyrillic": ("uk",), "Devanagari": ("hi",), "Telugu": ("te",), - "Greek": ("en",), + "Greek": ("el",), "Hebrew": ("en",), "Thai": ("en",), UNKNOWN_SCRIPT: ("en",), diff --git a/openmed/eval/golden/fixtures/i18n/uk.jsonl b/openmed/eval/golden/fixtures/i18n/uk.jsonl new file mode 100644 index 000000000..144a4205c --- /dev/null +++ b/openmed/eval/golden/fixtures/i18n/uk.jsonl @@ -0,0 +1 @@ +{"id": "golden-i18n-uk-clinical-pii", "language": "uk", "text": "Пацієнт: Олена Коваль. Дата народження 16.11.1975, телефон +380 67 123 45 67, РНОКПП 2974281300, адреса вулиця Хрещатик 22, поштовий індекс 01001.", "gold_spans": [{"start": 39, "end": 49, "label": "DATE", "text": "16.11.1975"}, {"start": 59, "end": 76, "label": "PHONE", "text": "+380 67 123 45 67"}, {"start": 85, "end": 95, "label": "ID_NUM", "text": "2974281300", "metadata": {"checksum_status": "valid", "identifier_type": "rnokpp"}}, {"start": 104, "end": 122, "label": "STREET_ADDRESS", "text": "вулиця Хрещатик 22"}, {"start": 140, "end": 145, "label": "ZIPCODE", "text": "01001"}], "metadata": {"category": "multilingual", "expected_output": {"method": "mask", "text": "Пацієнт: Олена Коваль. Дата народження [DATE], телефон [PHONE], РНОКПП [ID_NUM], адреса [STREET_ADDRESS], поштовий індекс [ZIPCODE]."}, "identifier_type": "rnokpp", "locale": "uk_UA", "synthetic": true}} diff --git a/openmed/service/client.py b/openmed/service/client.py index eb3cd9720..8dae1d0d5 100644 --- a/openmed/service/client.py +++ b/openmed/service/client.py @@ -30,6 +30,9 @@ "th", "ko", "ro", + "uk", + "cs", + "el", ] _DEFAULT_PII_MODEL = "OpenMed/OpenMed-PII-SuperClinical-Small-44M-v1" diff --git a/openmed/service/schemas.py b/openmed/service/schemas.py index 9b19817ac..a0f46885d 100644 --- a/openmed/service/schemas.py +++ b/openmed/service/schemas.py @@ -58,6 +58,9 @@ "th", "ko", "ro", + "uk", + "cs", + "el", ] diff --git a/openmed/training/synthetic/locale_phi.py b/openmed/training/synthetic/locale_phi.py index a655addc8..68450ba3a 100644 --- a/openmed/training/synthetic/locale_phi.py +++ b/openmed/training/synthetic/locale_phi.py @@ -18,9 +18,11 @@ from openmed.core.pii_i18n import ( SUPPORTED_LANGUAGES, validate_aadhaar, + validate_czech_rodne_cislo, validate_dutch_bsn, validate_french_nir, validate_german_steuer_id, + validate_greek_amka, validate_indonesian_nik, validate_israeli_teudat_zehut, validate_italian_codice_fiscale, @@ -30,6 +32,7 @@ validate_spanish_nie, validate_thai_national_id, validate_turkish_tckn, + validate_ukrainian_rnokpp, ) SUPPORTED_LOCALE_PHI_LANGUAGES: Final[tuple[str, ...]] = ( @@ -50,6 +53,9 @@ "th", "ko", "ro", + "uk", + "cs", + "el", ) LOCALE_PHI_LABELS: Final[tuple[str, ...]] = ( @@ -236,6 +242,33 @@ ". Control la ", ".", ), + "uk": ( + "Клінічна нотатка: пацієнт ", + " народився ", + ". РНОКПП пацієнта ", + ". Телефон ", + ". Адреса ", + ". Дата наступного огляду ", + ".", + ), + "cs": ( + "Klinická poznámka: pacient ", + " se narodil ", + ". Rodné číslo pacienta ", + ". Telefon ", + ". Adresa ", + ". Datum kontroly ", + ".", + ), + "el": ( + "Κλινική σημείωση: ασθενής ", + " γεννήθηκε στις ", + ". ΑΜΚΑ ασθενούς ", + ". Τηλέφωνο ", + ". Διεύθυνση ", + ". Ημερομηνία επανελέγχου ", + ".", + ), } _NATIONAL_ID_VALIDATORS: Final[Mapping[str, Callable[[str], bool]]] = { @@ -254,6 +287,9 @@ "th": validate_thai_national_id, "ko": validate_korean_rrn, "ro": validate_romanian_cnp, + "uk": validate_ukrainian_rnokpp, + "cs": validate_czech_rodne_cislo, + "el": validate_greek_amka, } _NATIONAL_ID_VALIDATOR_NAMES: Final[Mapping[str, str]] = { @@ -272,6 +308,9 @@ "th": "pii_i18n.validate_thai_national_id", "ko": "pii_i18n.validate_korean_rrn", "ro": "pii_i18n.validate_romanian_cnp", + "uk": "pii_i18n.validate_ukrainian_rnokpp", + "cs": "pii_i18n.validate_czech_rodne_cislo", + "el": "pii_i18n.validate_greek_amka", } diff --git a/tests/unit/core/test_id_provider_registry.py b/tests/unit/core/test_id_provider_registry.py index f48813a9e..a60bed9b2 100644 --- a/tests/unit/core/test_id_provider_registry.py +++ b/tests/unit/core/test_id_provider_registry.py @@ -39,6 +39,7 @@ def _always_valid(_text: str) -> bool: ("lv", "personas_kods"), ("ko", "rrn"), ("hu", "taj"), + ("uk", "rnokpp"), ("cs", "rodne_cislo"), ("sk", "rodne_cislo"), ("pt", "cpf"), @@ -69,6 +70,7 @@ def _always_valid(_text: str) -> bool: ("lv", "personas_kods", "lv_LV"), ("ko", "rrn", "ko_KR"), ("hu", "taj", "hu_HU"), + ("uk", "rnokpp", "uk_UA"), ("cs", "rodne_cislo", "cs_CZ"), ("sk", "rodne_cislo", "sk_SK"), ("pt", "cpf", "pt_BR"), diff --git a/tests/unit/core/test_script_detect.py b/tests/unit/core/test_script_detect.py index b5a31ec76..760ef4c17 100644 --- a/tests/unit/core/test_script_detect.py +++ b/tests/unit/core/test_script_detect.py @@ -76,6 +76,12 @@ def test_script_language_hints_cover_detectable_scripts(): assert set(hints) <= SUPPORTED_LANGUAGES | NATIONAL_ID_ONLY_LANGUAGES +def test_cee_script_language_hints_route_to_native_packs(): + assert "cs" in candidate_languages_for_script("Latin") + assert candidate_languages_for_script("Cyrillic") == ("uk",) + assert candidate_languages_for_script("Greek") == ("el",) + + def test_normalize_for_pii_detection_folds_obfuscation_with_offset_map(): text = "Patient J\u200bo\u0301hn D\u03bfe" normalized = normalize_for_pii_detection(text) diff --git a/tests/unit/test_pii_i18n.py b/tests/unit/test_pii_i18n.py index 6e9a6fe64..98082187c 100644 --- a/tests/unit/test_pii_i18n.py +++ b/tests/unit/test_pii_i18n.py @@ -68,6 +68,7 @@ validate_spanish_nie, validate_thai_national_id, validate_turkish_tckn, + validate_ukrainian_rnokpp, validate_vietnamese_cccd, validate_vietnamese_cmnd, ) @@ -99,6 +100,9 @@ def test_supported_languages(self): "th", "ko", "ro", + "uk", + "cs", + "el", } def test_national_id_only_languages(self): @@ -115,8 +119,6 @@ def test_national_id_only_languages(self): "hr", "bg", "fi", - "cs", - "el", "vi", } @@ -141,6 +143,9 @@ def test_language_model_prefix(self): assert LANGUAGE_MODEL_PREFIX["th"] == "Thai-" assert LANGUAGE_MODEL_PREFIX["ko"] == "Korean-" assert LANGUAGE_MODEL_PREFIX["ro"] == "Romanian-" + assert LANGUAGE_MODEL_PREFIX["uk"] == "Ukrainian-" + assert LANGUAGE_MODEL_PREFIX["cs"] == "Czech-" + assert LANGUAGE_MODEL_PREFIX["el"] == "Greek-" def test_default_pii_models_all_languages(self): assert set(DEFAULT_PII_MODELS.keys()) == SUPPORTED_LANGUAGES @@ -165,6 +170,9 @@ def test_default_pii_models_naming(self): == "OpenMed/OpenMed-PII-Korean-NomicMed-Large-395M-v1" ) assert DEFAULT_PII_MODELS["ro"] == "OpenMed/privacy-filter-multilingual" + assert DEFAULT_PII_MODELS["uk"] == "OpenMed/privacy-filter-multilingual" + assert DEFAULT_PII_MODELS["cs"] == "OpenMed/privacy-filter-multilingual" + assert DEFAULT_PII_MODELS["el"] == "OpenMed/privacy-filter-multilingual" # English has no language prefix assert "French" not in DEFAULT_PII_MODELS["en"] assert "German" not in DEFAULT_PII_MODELS["en"] @@ -3650,6 +3658,139 @@ def test_finnish_i18n_golden_fixture_deidentifies_with_no_leakage_offline(): assert span["text"] not in result.deidentified_text +def test_validate_ukrainian_rnokpp(): + assert validate_ukrainian_rnokpp("2974281300") + assert validate_ukrainian_rnokpp("3695007088") + + assert validate_ukrainian_rnokpp("2974281301") is False + assert validate_ukrainian_rnokpp("297428130") is False + assert validate_ukrainian_rnokpp("29742 81300") is False + assert validate_ukrainian_rnokpp("abcdefghij") is False + + +def test_generated_ukrainian_surrogate_passes_validator(): + assert LANG_TO_LOCALE["uk"] == "uk_UA" + + anonymizer = Anonymizer(lang="uk", consistent=True, seed=42) + surrogate = anonymizer.surrogate("2974281300", "national_id") + + assert validate_ukrainian_rnokpp(surrogate) is True + assert surrogate != "2974281300" + + +def test_ukrainian_clinical_sample_expected_spans(): + text = ( + "Пацієнт: Олена Коваль. Дата народження 16.11.1975, " + "телефон +380 67 123 45 67, РНОКПП 2974281300, " + "адреса вулиця Хрещатик 22, поштовий індекс 01001." + ) + expected = { + ("date", 39, 49, "16.11.1975"), + ("phone_number", 59, 76, "+380 67 123 45 67"), + ("national_id", 85, 95, "2974281300"), + ("street_address", 104, 122, "вулиця Хрещатик 22"), + ("postcode", 140, 145, "01001"), + } + observed = set() + for pattern in get_patterns_for_language("uk"): + for match in re.finditer(pattern.pattern, text, pattern.flags): + value = match.group(0) + if pattern.validator is not None and not pattern.validator(value): + continue + observed.add((pattern.entity_type, match.start(), match.end(), value)) + + assert expected <= observed + + +def test_ukrainian_textual_date_and_street_patterns(): + text = "Дата народження 16 листопада 1975, адреса проспект Свободи 15." + observed = { + (pattern.entity_type, match.group(0)) + for pattern in get_patterns_for_language("uk") + for match in re.finditer(pattern.pattern, text, pattern.flags) + } + + assert ("date", "16 листопада 1975") in observed + assert ("street_address", "проспект Свободи 15") in observed + + +def test_ukrainian_i18n_golden_fixture_offsets(): + fixture_path = Path("openmed/eval/golden/fixtures/i18n/uk.jsonl") + rows = [ + json.loads(line) + for line in fixture_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + assert len(rows) == 1 + row = rows[0] + assert row["language"] == "uk" + assert row["metadata"]["synthetic"] is True + assert row["metadata"]["category"] == "multilingual" + + text = row["text"] + expected = { + ("DATE", 39, 49, "16.11.1975"), + ("PHONE", 59, 76, "+380 67 123 45 67"), + ("ID_NUM", 85, 95, "2974281300"), + ("STREET_ADDRESS", 104, 122, "вулиця Хрещатик 22"), + ("ZIPCODE", 140, 145, "01001"), + } + actual = { + (span["label"], span["start"], span["end"], span["text"]) + for span in row["gold_spans"] + } + assert actual == expected + for label, start, end, value in actual: + assert text[start:end] == value, label + + identifier = next( + span["text"] for span in row["gold_spans"] if span["label"] == "ID_NUM" + ) + assert validate_ukrainian_rnokpp(identifier) + + +def test_ukrainian_i18n_golden_fixture_deidentifies_with_no_leakage_offline(): + from openmed.core.pii import ( + _apply_safety_sweep_to_result, + _build_deidentification_result, + ) + from openmed.processing.outputs import PredictionResult + + fixture_path = Path("openmed/eval/golden/fixtures/i18n/uk.jsonl") + row = json.loads(fixture_path.read_text(encoding="utf-8").strip()) + empty_result = PredictionResult( + text=row["text"], + entities=[], + model_name="offline-safety-sweep", + timestamp="2026-07-19T00:00:00Z", + metadata={}, + ) + + swept_result, added_count = _apply_safety_sweep_to_result( + row["text"], + empty_result, + lang="uk", + ) + result = _build_deidentification_result( + row["text"], + swept_result, + effective_method="mask", + keep_year=False, + date_shift_days=None, + keep_mapping=False, + lang="uk", + consistent=False, + seed=None, + locale=None, + use_safety_sweep=True, + ) + + assert added_count == len(row["gold_spans"]) + for span in row["gold_spans"]: + assert span["text"] not in result.deidentified_text + + def test_validate_czech_rodne_cislo(): # Modern ten-digit form (delegated to the Czechoslovak validator). assert validate_czech_rodne_cislo("751116/0008") diff --git a/tests/unit/training/test_synthetic_locale_phi.py b/tests/unit/training/test_synthetic_locale_phi.py index d2c1c8939..db299a7a6 100644 --- a/tests/unit/training/test_synthetic_locale_phi.py +++ b/tests/unit/training/test_synthetic_locale_phi.py @@ -8,9 +8,11 @@ from openmed.core.pii_i18n import ( SUPPORTED_LANGUAGES, validate_aadhaar, + validate_czech_rodne_cislo, validate_dutch_bsn, validate_french_nir, validate_german_steuer_id, + validate_greek_amka, validate_indonesian_nik, validate_israeli_teudat_zehut, validate_italian_codice_fiscale, @@ -20,6 +22,7 @@ validate_spanish_nie, validate_thai_national_id, validate_turkish_tckn, + validate_ukrainian_rnokpp, ) from openmed.training.synthetic import ( LOCALE_PHI_LABELS, @@ -44,6 +47,9 @@ "th": validate_thai_national_id, "ko": validate_korean_rrn, "ro": validate_romanian_cnp, + "uk": validate_ukrainian_rnokpp, + "cs": validate_czech_rodne_cislo, + "el": validate_greek_amka, } _SCRIPT_RANGES = { @@ -54,6 +60,8 @@ "ko": ("\uac00", "\ud7a3"), "te": ("\u0c00", "\u0c7f"), "th": ("\u0e00", "\u0e7f"), + "uk": ("\u0400", "\u04ff"), + "el": ("\u0370", "\u03ff"), } @@ -103,7 +111,7 @@ def test_locale_phi_generation_is_deterministic_per_seed(): assert first == second -@pytest.mark.parametrize("language", ("ar", "he", "hi", "ja", "te", "th")) +@pytest.mark.parametrize("language", ("ar", "he", "hi", "ja", "te", "th", "uk", "el")) def test_non_latin_locale_templates_render_target_script(language): example = LocalePhiGenerator(seed=29).generate(language) low, high = _SCRIPT_RANGES[language]