Fr tn

.gitattributes

@@ -0,0 +1 @@
+*.nemo filter=lfs diff=lfs merge=lfs -text

Jenkinsfile

@@ -16,7 +16,7 @@ pipeline {
     EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-14-23-0'
     ES_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     ES_EN_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-13-23-1'
-    FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
+    FR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-18-23-0'
     HU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     PT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'

nemo_text_processing/inverse_text_normalization/es/graph_utils.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+# Copyright 2015 and onwards Google, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +19,59 @@
 from pynini.lib import pynutil
 
 
+def int_to_roman(fst: 'pynini.FstLike') -> 'pynini.FstLike':
+    """
+    Alters given fst to convert Arabic numerals into Roman integers (lower cased). Valid for values up to 3999.
+    e.g.
+        "5" -> "v"
+        "treinta y uno" -> "xxxi"
+    Args:
+        fst: Any fst. Composes fst onto Roman conversion outputs.
+    """
+
+    def _load_roman(file: str):
+        roman_numerals = pynini.string_file(get_abs_path(file))
+        return pynini.invert(roman_numerals)
+
+    digit = _load_roman("data/roman/digit.tsv")
+    ties = _load_roman("data/roman/ties.tsv")
+    hundreds = _load_roman("data/roman/hundreds.tsv")
+    thousands = _load_roman("data/roman/thousands.tsv")
+
+    graph = (
+        digit
+        | ties + (digit | pynutil.add_weight(pynutil.delete("0"), 0.01))
+        | (
+            hundreds
+            + (ties | pynutil.add_weight(pynutil.delete("0"), 0.01))
+            + (digit | pynutil.add_weight(pynutil.delete("0"), 0.01))
+        )
+        | (
+            thousands
+            + (hundreds | pynutil.add_weight(pynutil.delete("0"), 0.01))
+            + (ties | pynutil.add_weight(pynutil.delete("0"), 0.01))
+            + (digit | pynutil.add_weight(pynutil.delete("0"), 0.01))
+        )
+    ).optimize()
+
+    return fst @ graph
+
+
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 def int_to_roman(fst: 'pynini.FstLike') -> 'pynini.FstLike':
     """
     Alters given fst to convert Arabic numerals into Roman integers (lower cased). Valid for values up to 3999.

nemo_text_processing/text_normalization/es/taggers/decimals.py

@@ -37,8 +37,7 @@ def get_quantity(decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLik
     """
     Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
     e.g. 2 millones -> integer_part: "dos" quantity: "millones"
-    e.g. 2,4 millones -> integer_part: "dos" fractional_part: "quatro" quantity: "millones"
-    e.g. 2,400 millones -> integer_part: "dos mil cuatrocientos" fractional_part: "quatro" quantity: "millones"
+    e.g. 2,4 millones -> integer_part: "dos" fractional_part: "cuatro" quantity: "millones"
 
     Args:
         decimal_graph: DecimalFST

nemo_text_processing/text_normalization/fr/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/dates/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/electronic/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/fractions/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/fractions/irregular_denominators.tsv

@@ -0,0 +1,7 @@
+premième	tout
+deuxième	demi
+deuxièmes	demi
+troisième	tiers
+troisièmes	tiers
+quatrième	quart
+quatrièmes	quarts

nemo_text_processing/text_normalization/fr/data/measures/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/money/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/numbers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+1	un
+2	deux
+3	trois
+4	quatre
+5	cinq
+6	six
+7	sept
+8	huit
+9	neuf

nemo_text_processing/text_normalization/fr/data/numbers/eighty.tsv

@@ -0,0 +1 @@
+8	quatre-vingt

nemo_text_processing/text_normalization/fr/data/numbers/ninety.tsv

@@ -0,0 +1 @@
+9	quatre-vingt

nemo_text_processing/text_normalization/fr/data/numbers/quantities.tsv

@@ -0,0 +1,10 @@
+million
+millions
+milliard
+milliards
+billion
+billions
+billiard
+billiards
+trillion
+trillions

nemo_text_processing/text_normalization/fr/data/numbers/seventy.tsv

@@ -0,0 +1 @@
+7	soixante

nemo_text_processing/text_normalization/fr/data/numbers/teens.tsv

@@ -0,0 +1,9 @@
+11	onze
+12	douze
+13	treize
+14	quatorze
+15	quinze
+16	seize
+17	dix-sept
+18	dix-huit
+19	dix-neuf

nemo_text_processing/text_normalization/fr/data/numbers/ten.tsv

@@ -0,0 +1 @@
+10	dix

nemo_text_processing/text_normalization/fr/data/numbers/tens_simple.tsv

@@ -0,0 +1,5 @@
+2	vingt
+3	trente
+4	quarante
+5	cinquante
+6	soixante

nemo_text_processing/text_normalization/fr/data/numbers/zero.tsv

@@ -0,0 +1 @@
+0	zéro

nemo_text_processing/text_normalization/fr/data/ordinals/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/ordinals/irregular_numbers.tsv

@@ -0,0 +1,29 @@
+quatre	quatrième
+cinq	cinquième
+neuf	neuvième
+onze	onzième
+douze	douzième
+treize	treizième
+quatorze	quatorzième
+quinze	quinzième
+seize	seizième
+dix-neuf	dix neuvième
+trente	trentième
+quarante	quarantième
+cinquante	cinquantième
+soixante	soixantième
+soixante-onze	soixante onzième
+soixante-douze	soixante douzième
+soixante-treize	soixante treizième
+soixante-quatorze	soixante quatorzième
+soixante-quinze	soixante quinzième
+soixante-seize	soixante seizième
+soixante-dix-neuf	soixante dix neuvième
+quatre-vingt-onze	quatre vingt onzième
+quatre-vingt-douze	quatre vingt douzième
+quatre-vingt-treize	quatre vingt treizième
+quatre-vingt-quatorze	quatre vingt quatorzième
+quatre-vingt-quinze	quatre vingt quinzième
+quatre-vingt-seize	quatre vingt seizième
+quatre-vingt-dix-neuf	quatre vingt dix neuvième
+mille	millième

nemo_text_processing/text_normalization/fr/data/ordinals/suffixes.tsv

@@ -0,0 +1,5 @@
+e	ième
+ème	ième
+°	ième
+er	ier
+ère	ière

nemo_text_processing/text_normalization/fr/data/roman/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/telephone/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/data/time/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/fr/taggers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.