Jp itn update 240805

Jenkinsfile

@@ -26,7 +26,7 @@ pipeline {
     IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
     HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
-    JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-15-24-0'
+    JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-27-24-0'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/inverse_text_normalization/ja/graph_utils.py

@@ -29,13 +29,14 @@
 
 NEMO_CHAR = utf8.VALID_UTF8_CHAR
 
+NEMO_NARROW_NON_BREAK_SPACE = "\u202F"
 NEMO_DIGIT = byte.DIGIT
 NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize()
 NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize()
 NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize()
 NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize()
 NEMO_HEX = pynini.union(*string.hexdigits).optimize()
-NEMO_NON_BREAKING_SPACE = u"\u00A0"
+NEMO_NON_BREAKING_SPACE = "\u00A0"
 NEMO_SPACE = " "
 NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00A0").optimize()
 NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize()
@@ -45,6 +46,7 @@
 NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize()
 
 NEMO_SIGMA = pynini.closure(NEMO_CHAR)
+
 NEMO_NOT_ALPHA = pynini.difference(NEMO_SIGMA, NEMO_ALPHA).optimize()
 NEMO_LOWER_NOT_A = pynini.union(
     "b",

nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py

@@ -37,13 +37,15 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
         decimal = decimal.just_decimal
 
         fraction_word = pynutil.delete("分の") | pynutil.delete(" 分 の　") | pynutil.delete("分 の　") | pynutil.delete("分 の")
-        integer_word = pynini.accep("と") | pynini.accep("荷")
-        optional_sign = (
+
+        integer_word = pynutil.delete("と") | pynutil.delete("荷")
+        root_word = pynini.accep("√") | pynini.cross("ルート", "√")
+
+        graph_sign = (
             pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"")
         )
 
-        root_word = pynini.accep("√") | pynini.cross("ルート", "√")
-        root_integer = (
+        graph_integer = (
             pynutil.insert("integer_part: \"")
             + (
                 (decimal | decimal + integer_word)
@@ -54,15 +56,16 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
             + pynutil.insert("\"")
         )
 
-        root_denominator = (
+        graph_denominator = (
             pynutil.insert("denominator: \"")
             + (
                 ((decimal) | (cardinal + root_word + cardinal) | (root_word + cardinal) | cardinal)
                 + pynini.closure(pynutil.delete(' '), 0, 1)
             )
             + pynutil.insert("\"")
         )
-        root_numerator = (
+
+        graph_numerator = (
             pynutil.insert("numerator: \"")
             + (
                 pynini.closure(pynutil.delete(' '))
@@ -71,26 +74,30 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
             + pynutil.insert("\"")
         )
 
-        graph_root_fraction = (
-            pynini.closure((optional_sign + pynutil.insert(" ")), 0, 1)
-            + root_denominator
+        graph_fraction_sign = (
+            graph_sign
+            + pynutil.insert(" ")
+            + graph_denominator
             + pynutil.insert(" ")
             + fraction_word
-            + root_numerator
+            + graph_numerator
         )
 
-        graph_root_with_integer = (
-            pynini.closure((optional_sign + pynutil.insert(" ")), 0, 1)
-            + root_integer
-            # + inetegr_word
+        graph_fraction_no_sign = graph_denominator + pynutil.insert(" ") + fraction_word + graph_numerator
+
+        graph_regular_fractions = graph_fraction_sign | graph_fraction_no_sign
+
+        graph_integer_fraction_sign = (
+            pynini.closure((graph_sign + pynutil.insert(" ")), 0, 1)
+            + pynutil.add_weight(graph_integer, 1.1)
             + pynutil.insert(" ")
-            + root_denominator
+            + graph_denominator
             + pynutil.insert(" ")
             + fraction_word
-            + root_numerator
+            + graph_numerator
         )
 
-        final_graph = graph_root_fraction | graph_root_with_integer
+        final_graph = graph_regular_fractions | graph_integer_fraction_sign
 
         final_graph = self.add_tokens(final_graph)
         self.fst = final_graph.optimize()

nemo_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py

@@ -91,11 +91,11 @@ def __init__(
 
             classify = (
                 pynutil.add_weight(whitelist_graph, 1.01)
-                | pynutil.add_weight(cardinal_graph, 1.0)  # was -1.1
+                | pynutil.add_weight(cardinal_graph, 1.0)
                 | pynutil.add_weight(ordinal_graph, 1.1)
                 | pynutil.add_weight(date_graph, 1.1)
                 | pynutil.add_weight(decimal_graph, 1.1)
-                | pynutil.add_weight(fraction_graph, 1.1)
+                | pynutil.add_weight(fraction_graph, 1.0)
                 | pynutil.add_weight(time_graph, 1.0)
                 | pynutil.add_weight(word_graph, 100)
                 | pynutil.add_weight(punct_graph, 1.1)

nemo_text_processing/inverse_text_normalization/ja/taggers/word.py

@@ -12,10 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst
+from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst
 
 
 class WordFst(GraphFst):
@@ -26,5 +25,5 @@ class WordFst(GraphFst):
 
     def __init__(self):
         super().__init__(name="word", kind="classify")
-        word = pynutil.insert("name: \"") + NEMO_NOT_QUOTE + pynutil.insert("\"")
+        word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"")
         self.fst = word.optimize()

nemo_text_processing/inverse_text_normalization/ja/utils.py

@@ -21,7 +21,15 @@ def get_abs_path(rel_path):
 
     Args:
         rel_path: relative path to this file
+<<<<<<< HEAD
+<<<<<<< HEAD
 
+=======
+
+>>>>>>> 0a4a21c (Jp itn 20240221 (#141))
+=======
+
+>>>>>>> 59f46198ab4c8880c6a5fb88f3cbee9530156498
     Returns absolute path
     """
     return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path

nemo_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py

@@ -16,7 +16,11 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.inverse_text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst
+from nemo_text_processing.inverse_text_normalization.ja.graph_utils import (
+    NEMO_NON_BREAKING_SPACE,
+    NEMO_NOT_QUOTE,
+    GraphFst,
+)
 
 
 class FractionFst(GraphFst):
@@ -33,34 +37,29 @@ def __init__(self):
         """
         super().__init__(name="fraction", kind="verbalize")
 
-        sign_component = pynutil.delete("negative: \"") + pynini.closure("-") + pynutil.delete("\"")
+        sign_component = pynutil.delete("negative: \"") + pynini.closure("-", 1) + pynutil.delete("\"")
 
-        # integer_component = (
-        #     pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")
-        # ) | (
-        #     sign_component
-        #     + pynutil.delete(" ")
-        #     + pynutil.delete("integer_part: \"")
-        #     + pynini.closure(NEMO_NOT_QUOTE)
-        #     + pynutil.delete("\"")
-        # )
+        integer_component = (
+            pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        )
 
-        integer_component = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")
         denominator_component = (
-            pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")
+            pynutil.delete("denominator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
         )
-        numerator_component = pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE) + pynutil.delete("\"")
 
-        final_graph = (
-            pynini.closure(sign_component, 0, 1)
-            + pynutil.delete(" ")
-            + pynini.closure(integer_component + pynutil.delete(" "))
-            # + pynini.closure(sign_component + pynutil.delete(" "))
+        numerator_component = (
+            pynutil.delete("numerator: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        )
+
+        regular_graph = (
+            pynini.closure((sign_component + pynutil.delete(" ")), 0, 1)
+            + pynini.closure(integer_component + pynutil.delete(" ") + pynutil.insert(NEMO_NON_BREAKING_SPACE))
             + numerator_component
             + pynutil.delete(" ")
             + pynutil.insert("/")
             + denominator_component
         )
 
-        final_graph = self.delete_tokens(final_graph)
+        final_graph = self.delete_tokens(regular_graph)
+
         self.fst = final_graph.optimize()