Jp tn 20241017 (#240)

* ja tn

Signed-off-by: Alex Cui <[email protected]>

* adding ja

Signed-off-by: Alex Cui <[email protected]>

* removing

Signed-off-by: Alex Cui <[email protected]>

* updated tests

Signed-off-by: Alex Cui <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* addressing comment

Signed-off-by: Alex Cui <[email protected]>

* addressing ci

Signed-off-by: Alex Cui <[email protected]>

* addressing ci

Signed-off-by: Alex Cui <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* addresing comment

Signed-off-by: Alex Cui <[email protected]>

* removing

Signed-off-by: Alex Cui <[email protected]>

* adresing comment

Signed-off-by: Alex Cui <[email protected]>

* removing unused import

Signed-off-by: Alex Cui <[email protected]>

* addressing comment

Signed-off-by: Alex Cui <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* addressing comment;

Signed-off-by: Alex Cui <[email protected]>

* addressing comment

Signed-off-by: Alex Cui <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* date for ja

Signed-off-by: Alex Cui <[email protected]>

* addresing comment

Signed-off-by: Alex Cui <[email protected]>

* addressing comment

Signed-off-by: Alex Cui <[email protected]>

* jenkins

Signed-off-by: Alex Cui <[email protected]>

* addresing comment

Signed-off-by: Alex Cui <[email protected]>

* addressing comment

Signed-off-by: Alex Cui <[email protected]>

* typo

Signed-off-by: Alex Cui <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* adressing comment

Signed-off-by: Alex Cui <[email protected]>

* addressing comment

Signed-off-by: Alex Cui <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* ci

Signed-off-by: Alex Cui <[email protected]>

---------

Signed-off-by: Alex Cui <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Jenkinsfile

@@ -26,7 +26,7 @@ pipeline {
     IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-22-24-0'
     HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
-    JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-27-24-0'
+    JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/ja/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst
+from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst
+from nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst

nemo_text_processing/text_normalization/ja/data/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ja/data/date/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ja/data/date/day.tsv

@@ -0,0 +1,31 @@
+1	一
+2	二
+3	三
+4	四
+5	五
+6	六
+7	七
+8	八
+9	九
+10	十
+11	十一
+12	十二
+13	十三
+14	十四
+15	十五
+16	十六
+17	十七
+18	十八
+19	十九
+20	二十
+21	二十一
+22	二十二
+23	二十三
+24	二十四
+25	二十五
+26	二十六
+27	二十七
+28	二十八
+29	二十九
+30	三十
+31	三十一

nemo_text_processing/text_normalization/ja/data/date/era.tsv

@@ -0,0 +1,12 @@
+令和
+平成
+昭和
+大正
+明治
+西暦
+和暦
+西洋暦
+グレゴリオ暦
+紀元前
+紀元
+紀元後

nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv

@@ -0,0 +1,5 @@
+R.	令和
+H.	平成
+S.	昭和
+T.	大正
+M.	明治

nemo_text_processing/text_normalization/ja/data/date/month.tsv

@@ -0,0 +1,12 @@
+1	一
+2	二
+3	三
+4	四
+5	五
+6	六
+7	七
+8	八
+9	九
+10	十
+11	十一
+12	十二

nemo_text_processing/text_normalization/ja/data/date/week.tsv

@@ -0,0 +1,15 @@
+月	月曜日
+火	火曜日
+水	水曜日
+木	木曜日
+金	金曜日
+土	土曜日
+日	日曜日
+祝	祝日
+月曜日
+火曜日
+水曜日
+木曜日
+金曜日
+土曜日
+日曜日

nemo_text_processing/text_normalization/ja/data/numbers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+1	一
+2	二
+3	三
+4	四
+5	五
+6	六
+7	七
+8	八
+9	九

nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv

@@ -0,0 +1,10 @@
+10	十
+11	十一
+12	十二
+13	十三
+14	十四
+15	十五
+16	十六
+17	十七
+18	十八
+19	十九

nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv

@@ -0,0 +1,8 @@
+2	二十
+3	三十
+4	四十
+5	五十
+6	六十
+7	七十
+8	八十
+9	九十

nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv

@@ -0,0 +1 @@
+0	零

nemo_text_processing/text_normalization/ja/data/symbol.tsv

@@ -0,0 +1,23 @@
+&	アンド
+#	ハッシュタグ
+@	アット
+§	セクション
+™	トレードマーク
+®	登録商標マーク
+©	著作権
+_	アンダースコア
+%	パーセント
+*	星印
++	プラス
+/	スラッシュ
+=	エコール
+^	曲折アクセント記号
+|	縦棒
+~	ティルダ
+$	ドール
+£	ポンド
+€	ユーロ
+₩	ウォン
+¥	円
+°	度
+º	度

nemo_text_processing/text_normalization/ja/data/time/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ja/data/time/division.tsv

@@ -0,0 +1,23 @@
+今朝
+今夜
+今晩
+午前
+午後
+夕方
+夜中
+夜
+夜半
+朝
+早朝
+明け方
+昼
+深夜
+毎朝
+毎夜
+毎晩
+毎日
+真夜中
+翌日
+未明
+正午
+真夜中の

nemo_text_processing/text_normalization/ja/data/time/hour.tsv

@@ -0,0 +1,24 @@
+1	一
+2	二
+3	三
+4	四
+5	五
+6	六
+7	七
+8	八
+9	九
+10	十
+11	十一
+12	十二
+13	十三
+14	十四
+15	十五
+16	十六
+17	十七
+18	十八
+19	十九
+20	二十
+21	二十一
+22	二十二
+23	二十三
+24	二十四

nemo_text_processing/text_normalization/ja/data/time/minute.tsv

@@ -0,0 +1,60 @@
+1	一
+2	二
+3	三
+4	四
+5	五
+6	六
+7	七
+8	八
+9	九
+10	十
+11	十一
+12	十二
+13	十三
+14	十四
+15	十五
+16	十六
+17	十七
+18	十八
+19	十九
+20	二十
+21	二十一
+22	二十二
+23	二十三
+24	二十四
+25	二十五
+26	二十六
+27	二十七
+28	二十八
+29	二十九
+30	三十
+31	三十一
+32	三十二
+33	三十三
+34	三十四
+35	三十五
+36	三十六
+37	三十七
+38	三十八
+39	三十九
+40	四十
+41	四十一
+42	四十二
+43	四十三
+44	四十四
+45	四十五
+46	四十六
+47	四十七
+48	四十八
+49	四十九
+50	五十
+51	五十一
+52	五十二
+53	五十三
+54	五十四
+55	五十五
+56	五十六
+57	五十七
+58	五十八
+59	五十九
+60	六十