Add Nepali (npi-Deva) support

README.md

@@ -291,6 +291,7 @@ A few notes are in order regarding this data structure:
 | nhi-Latn    | Western Sierra Puebla Nahuatl        |
 | nld-Latn    | Dutch                   |
 | nno-Latn    | Norwegian (Nynorsk)     |
+| npi-Deva    | Nepali                  |
 | nya-Latn    | Chichewa                |
 | ood-Latn-alv| Tohono O'odham (Alvarez–Hale) |
 | ood-Latn-sax| Tohono O'odham (Saxton) |

epitran/__init__.py

@@ -1,9 +1,8 @@
-from importlib.metadata import version, PackageNotFoundError
+from importlib.metadata import PackageNotFoundError, version
 
 from epitran._epitran import Epitran
 from epitran.reromanize import ReRomanizer
 
-
 try:
     __version__ = version("epitran")
 except PackageNotFoundError:

epitran/_epitran.py

@@ -1,8 +1,8 @@
-# -*- coding: utf-8 -*-
 import logging
 
 import panphon.featuretable
-from epitran.epihan import Epihan, EpihanTraditional, EpiJpan, EpiCanto
+
+from epitran.epihan import EpiCanto, Epihan, EpihanTraditional, EpiJpan
 from epitran.flite import FliteLexLookup
 from epitran.puncnorm import PuncNorm
 from epitran.simple import SimpleEpitran
@@ -22,7 +22,7 @@
 }
 
 
-class Epitran(object):
+class Epitran:
     """Unified interface for IPA transliteration/transcription
 
     :param code str: ISO 639-3 plus "-" plus ISO 15924 code of the language/script pair that should be loaded
@@ -103,7 +103,7 @@ def trans_list(self, word: str, normpunc: bool=False, ligatures: bool=False) ->
         """
         return self.ft.segs_safe(self.epi.transliterate(word, normpunc, ligatures))
 
-    def trans_delimiter(self, text: str, delimiter: str=str(' '), normpunc: bool=False, ligatures: bool=False) -> str:
+    def trans_delimiter(self, text: str, delimiter: str=' ', normpunc: bool=False, ligatures: bool=False) -> str:
         """Return IPA transliteration with a delimiter between segments
 
         :param text str: An orthographic text

epitran/backoff.py

@@ -1,18 +1,19 @@
-# -*- coding: utf-8 -*-
 
-from typing import List, Optional, Union
-import regex as re
-from . import _epitran
+
 import panphon.featuretable
+import regex as re
+
 from epitran.puncnorm import PuncNorm
-from epitran.xsampa import XSampa
 from epitran.stripdiacritics import StripDiacritics
+from epitran.xsampa import XSampa
+
+from . import _epitran
 
 
-class Backoff(object):
+class Backoff:
     """Implements rudimentary language ID and backoff."""
 
-    def __init__(self, lang_script_codes: List[str], cedict_file: Optional[str] = None) -> None:
+    def __init__(self, lang_script_codes: list[str], cedict_file: str | None = None) -> None:
         """Construct a Backoff object.
 
         Args:
@@ -61,7 +62,7 @@ def transliterate(self, token: str) -> str:
                     token = token[1:]
         return ''.join(tr_list)
 
-    def trans_list(self, token: str) -> List[str]:
+    def trans_list(self, token: str) -> list[str]:
         """Transliterate/transcribe a word into list of IPA phonemes.
 
         Args:
@@ -72,7 +73,7 @@ def trans_list(self, token: str) -> List[str]:
         """
         return self.ft.segs_safe(self.transliterate(token))
 
-    def xsampa_list(self, token: str) -> Union[str, List[str]]:
+    def xsampa_list(self, token: str) -> str | list[str]:
         """Transcribe a word into a list of X-SAMPA phonemes.
 
         Args:

epitran/bin/connl2engipaspace.py

@@ -2,15 +2,14 @@
 
 import argparse
 import codecs
+import csv
 import logging
 from collections import Counter
-from typing import List
 
-import csv
+import panphon
 
 import epitran
 import epitran.flite
-import panphon
 
 logger = logging.getLogger('epitran')
 
@@ -50,7 +49,7 @@ def add_file(flite: epitran.flite.Flite, ft: panphon.FeatureTable, fn: str) -> C
             if len(fields) > 0:
                 orth = fields[0]
                 space.update(add_record(flite, ft, orth))
-    logger.debug(u'Length of counter:\t{}'.format(len(space)))
+    logger.debug(f'Length of counter:\t{len(space)}')
     return space
 
 
@@ -62,12 +61,12 @@ def print_space(output: str, space: Counter[str]) -> None:
             writer.writerow((i, char))
 
 
-def main(infiles: List[str], output: str) -> None:
+def main(infiles: list[str], output: str) -> None:
     flite = epitran.flite.Flite()
     ft = panphon.FeatureTable()
     space: Counter[str] = Counter()
     for fn in infiles:
-        logger.debug(u'Scanning:\t{}'.format(fn).encode('utf-8'))
+        logger.debug(f'Scanning:\t{fn}'.encode())
         space.update(add_file(flite, ft, fn))
     print_space(output, space)
 

epitran/bin/connl2ipaspace.py

@@ -2,13 +2,13 @@
 
 import argparse
 import codecs
+import csv
 import logging
 from collections import Counter
-from typing import List
 
-import epitran
 import panphon
-import csv
+
+import epitran
 
 logger = logging.getLogger('epitran')
 
@@ -45,7 +45,7 @@ def add_file_gen(epi: epitran.Epitran, ft: panphon.FeatureTable, fn: str) -> Cou
             if len(fields) > 0:
                 orth = fields[0]
                 space.update(add_record_gen(epi, ft, orth))
-    logger.debug(u'Length of counter:\t{}'.format(len(space)))
+    logger.debug(f'Length of counter:\t{len(space)}')
     return space
 
 
@@ -68,7 +68,7 @@ def add_file_op(epi: epitran.Epitran, ft: panphon.FeatureTable, fn: str) -> Coun
                         else:
                             space[trans[0]] += 1
                         trans = trans[1:]
-    logger.debug(u'Length of counter:\t{}'.format(len(space)))
+    logger.debug(f'Length of counter:\t{len(space)}')
     return space
 
 
@@ -80,12 +80,12 @@ def print_space(output: str, space: Counter[str]) -> None:
             writer.writerow((i, char))
 
 
-def main(code: str, op: bool, infiles: List[str], output: str) -> None:
+def main(code: str, op: bool, infiles: list[str], output: str) -> None:
     epi = epitran.Epitran(code)
     ft = panphon.FeatureTable()
     space: Counter[str] = Counter()
     for fn in infiles:
-        logger.debug(u'Scanning:\t{}'.format(fn).encode('utf-8'))
+        logger.debug(f'Scanning:\t{fn}'.encode())
         add_file = add_file_op if op else add_file_gen
         space.update(add_file(epi, ft, fn))
     print_space(output, space)

epitran/bin/decompose.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-import unicodedata
 import sys
+import unicodedata
 
 
 def main(fn: str) -> None:

epitran/bin/detectcaps.py

@@ -1,21 +1,21 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
-import unicodedata
 import fileinput
+import unicodedata
 
 
 def main() -> None:
-    for line in fileinput.input():
-        token = line.strip()
-        if len(token) > 1 and unicodedata.category(token[1]) == 'Lu':
-            is_cap = 0
-        elif len(token) > 0 and unicodedata.category(token[0]) == 'Lu':
-            is_cap = 1
-        else:
-            is_cap = 0
-        line = '{}\t{}'.format(is_cap, token)
-        print(line)
+    with fileinput.input() as f:
+        for line in f:
+            token = line.strip()
+            if len(token) > 1 and unicodedata.category(token[1]) == 'Lu':
+                is_cap = 0
+            elif len(token) > 0 and unicodedata.category(token[0]) == 'Lu':
+                is_cap = 1
+            else:
+                is_cap = 0
+            line = f'{is_cap}\t{token}'
+            print(line)
 
 
 if __name__ == '__main__':

epitran/bin/epitranscribe.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
+import argparse
 import sys
 import unicodedata
+
 import epitran
-import argparse
 
 
 def main(code: str) -> None:

epitran/bin/extract_test_cases.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python
 #
-import re
-import glob
 import csv
+import glob
+import re
 
-RE_DERIVATION = re.compile(r"""self\._?derivation\(u?['"]([^'"]+)['"], u?['"]([^'"]+)['"]\)""", re.M | re.S)
-RE_TR = re.compile(r"""self\.epi\.transliterate\(['"]([^'"]+)['"]\).+?self\.assertEqual\(tr, ['"]([^'"]+)['"]\)""", re.M | re.S)
-RE_RES = re.compile(r"""self\.epi\.transliterate\(['"]([^'"]+)['"]\).+?self\.assertEqual\(res, ['"]([^'"]+)['"]\)""", re.M | re.S)
-RE_ASSERT_TRANS = re.compile(r"""self\._assert_trans\(['"]([^'"]+)['"],\s*['"]([^'"]+)['"]\)""", re.M | re.S)
-RE_TUPLE = re.compile(r"""\(['"]([^'"]+)['"],\s*['"]([^'"]+)['"]\)""", re.M | re.S)
+RE_DERIVATION = re.compile(r"""self\._?derivation\(u?['"]([^'"]+)['"], u?['"]([^'"]+)['"]\)""", re.MULTILINE | re.DOTALL)
+RE_TR = re.compile(r"""self\.epi\.transliterate\(['"]([^'"]+)['"]\).+?self\.assertEqual\(tr, ['"]([^'"]+)['"]\)""", re.MULTILINE | re.DOTALL)
+RE_RES = re.compile(r"""self\.epi\.transliterate\(['"]([^'"]+)['"]\).+?self\.assertEqual\(res, ['"]([^'"]+)['"]\)""", re.MULTILINE | re.DOTALL)
+RE_ASSERT_TRANS = re.compile(r"""self\._assert_trans\(['"]([^'"]+)['"],\s*['"]([^'"]+)['"]\)""", re.MULTILINE | re.DOTALL)
+RE_TUPLE = re.compile(r"""\(['"]([^'"]+)['"],\s*['"]([^'"]+)['"]\)""", re.MULTILINE | re.DOTALL)
 RE_CODE = re.compile("""["']([a-z]{3}-[A-Z][a-z]{3})["']""")
 
 def extract_code(code: str) -> str:

epitran/bin/isbijective.py

@@ -1,29 +1,27 @@
 #!/usr/bin/env pythoh
 
-import glob
-from typing import List, Tuple
-
 import csv
+import glob
 
 
-def read_map(fn: str) -> List[Tuple[str, str]]:
+def read_map(fn: str) -> list[tuple[str, str]]:
     with open(fn, 'r', encoding='utf-8') as f:
         reader = csv.reader(f)
         next(reader)
         return [(a, b) for [a, b] in reader]
 
 
-def is_bijection(mapping: List[Tuple[str, str]]) -> bool:
+def is_bijection(mapping: list[tuple[str, str]]) -> bool:
     a, b = zip(*mapping)
     distinct_a, distinct_b = set(a), set(b)
     return len(distinct_a) == len(mapping) and len(distinct_b) == len(mapping)
 
 
-def main(map_fns: List[str]) -> None:
+def main(map_fns: list[str]) -> None:
     for fn in map_fns:
         mapping = read_map(fn)
         is_b = is_bijection(mapping)
-        print('{}\t{}'.format(fn, is_b))
+        print(f'{fn}\t{is_b}')
 
 
 if __name__ == '__main__':

epitran/bin/ltf2ipaspace.py

@@ -1,25 +1,23 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
 import argparse
+import csv
 import glob
 import os.path
-from typing import List, Set
 
+import panphon.featuretable
 from lxml import etree
-import csv
 
 import epitran
-import panphon.featuretable
 
 
-def read_tokens(fn: str) -> List[str]:
+def read_tokens(fn: str) -> list[str]:
     tree = etree.parse(fn)
     root = tree.getroot()
     return [tok.text for tok in root.findall('.//TOKEN')]
 
 
-def read_input(input_: List[List[str]], langscript: str) -> Set[str]:
+def read_input(input_: list[list[str]], langscript: str) -> set[str]:
     space = set()
     epi = epitran.Epitran(langscript)
     ft = panphon.featuretable.FeatureTable()
@@ -32,14 +30,14 @@ def read_input(input_: List[List[str]], langscript: str) -> Set[str]:
     return space
 
 
-def write_output(output: str, space: Set[str]) -> None:
+def write_output(output: str, space: set[str]) -> None:
     with open(output, 'w', newline='', encoding='utf-8') as f:
         writer = csv.writer(f)
-        for n, ch in enumerate(sorted(list(space))):
+        for n, ch in enumerate(sorted(space)):
             writer.writerow((n, ch))
 
 
-def main(langscript: str, input_: List[List[str]], output: str) -> None:
+def main(langscript: str, input_: list[list[str]], output: str) -> None:
     space = read_input(input_, langscript)
     write_output(output, space)
 

epitran/bin/migraterules.py

@@ -1,20 +1,17 @@
 #!/usr/bin/env Python
-# -*- coding: utf-8 -*-
 
 
 import csv
 import glob
 import re
-from typing import List, Optional
 
 
-
-def build_rule(fields: List[str]) -> Optional[str]:
+def build_rule(fields: list[str]) -> str | None:
     try:
         a, b, X, Y = fields
         b = "0" if not b else b
         a = "0" if not a else a
-        return '{} -> {} / {} _ {}'.format(a, b, X, Y)
+        return f'{a} -> {b} / {X} _ {Y}'
     except ValueError:
         print('Malformed rule: {}'.format(','.join(fields)))
         return None

epitran/bin/reromanize.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
 
-import epitran.reromanize
 import argparse
 import sys
 
+import epitran.reromanize
+
+
 def main(code, table):
     rr = epitran.reromanize.ReRomanizer(code, table)
     for line in sys.stdin: