Proof of concept of using polib and how to silent padpo on false posi…

…tives

POC in relation with AFPy#61

polib:
* replacement of padpo parser by polib parser is done with minimal change of
  the structure of padpo. Refactoring would be welcome.
* linelength checker is deleted as polib does not provide msgstr in vanilla
  form.
  Job can be done by powrap
* changes validated only on test files!

silent padpo:
* 2 tags are provided as a POC: 1 for glossary and 1 for nbsp
* polib is needed as padpo parser does not provide access to 'translator
  comment' field.

padpo/checkers/__init__.py

@@ -3,14 +3,12 @@
 from padpo.checkers.fuzzy import FuzzyChecker
 from padpo.checkers.glossary import GlossaryChecker
 from padpo.checkers.grammalecte import GrammalecteChecker
-from padpo.checkers.linelength import LineLengthChecker
 from padpo.checkers.nbsp import NonBreakableSpaceChecker
 
 checkers = [
     EmptyChecker(),
     FuzzyChecker(),
     GrammalecteChecker(),
     GlossaryChecker(),
-    LineLengthChecker(),
     NonBreakableSpaceChecker(),
 ]

padpo/checkers/baseclass.py

@@ -26,6 +26,17 @@ def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
         return NotImplementedError
 
+    def add_warning(self, level: str, item: PoItem, text: str, prefix:str=None, match:str=None, suffix:str=None):
+        silent_method_names = [self.__getattribute__(name) for name in self.__dir__() if name.startswith('silent_')]
+        if any([m(item, text, prefix, match, suffix) for m in silent_method_names if callable(m)]):
+                return
+        if level == 'warning':
+            item.add_warning(self.name, text)
+        else:
+            item.add_error(self.name, text)
+
+
+
 
 def replace_quotes(match):
     """Replace match with « xxxxxxx »."""

padpo/checkers/empty.py

@@ -12,5 +12,5 @@ class EmptyChecker(Checker):
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
-        if item.msgid_full_content and not item.msgstr_full_content:
+        if item.entry.msgid and not item.entry.msgstr:
             item.add_warning(self.name, "This entry is not translated yet.")

padpo/checkers/fuzzy.py

@@ -11,5 +11,5 @@ class FuzzyChecker(Checker):
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
-        if item.fuzzy:
+        if item.entry.fuzzy:
             item.add_warning(self.name, "This entry is tagged as fuzzy.")

padpo/checkers/glossary.py

@@ -10,14 +10,20 @@ class GlossaryChecker(Checker):
     """Checker for glossary usage."""
 
     name = "Glossary"
+    silent_re = re.compile('silent-glossary:`([^`]*)`')
+
+    def silent_glossary(self, item: PoItem, text: str, prefix, match, suffix):
+        for term in self.silent_re.findall(item.entry.tcomment):
+            if term in text[:text.find("that is not translated")]:
+                return True
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
-        if not item.msgstr_full_content:
+        if not item.entry.msgstr:
             return  # no warning
         original_content = item.msgid_rst2txt.lower()
         original_content = re.sub(r"« .*? »", "", original_content)
-        translated_content = item.msgstr_full_content.lower()
+        translated_content = item.entry.msgstr.lower()
         for word, translations in glossary.items():
             if re.match(fr"\b{word.lower()}\b", original_content):
                 for translated_word in translations:
@@ -30,10 +36,9 @@ def check_item(self, item: PoItem):
                         possibilities += '" or "'
                     possibilities += translations[-1]
                     possibilities += '"'
-                    item.add_warning(
-                        self.name,
+                    self.add_warning('warning', item,
                         f'Found "{word}" that is not translated in '
-                        f"{possibilities} in ###{item.msgstr_full_content}"
+                        f"{possibilities} in ###{item.entry.msgstr}"
                         "###.",
                     )
 

padpo/checkers/nbsp.py

@@ -10,6 +10,15 @@ class NonBreakableSpaceChecker(Checker):
     """Checker for missing non breakable spaces."""
 
     name = "NBSP"
+    after_silent_re=re.compile('silent-nbsp-after:`([^`]*)`')
+
+
+    def silent_nbsp_after(self, item: PoItem, text: str, prefix: str, match: str, suffix: str):
+        list_ok = self.after_silent_re.findall(item.entry.tcomment)
+        for term in list_ok:
+            if term in prefix:
+                return True
+
 
     def check_item(self, item: PoItem):
         """Check an item in a `*.po` file."""
@@ -31,16 +40,14 @@ def check_item(self, item: PoItem):
                     self.__add_message_space_before(item, prefix, match, suffix)
 
     def __add_message(self, item, prefix, match, suffix):
-        item.add_error(
-            self.name,
+        self.add_warning('error', item,
             "Space should be replaced with a non-breakable space in "
-            f'"{match}": between ###{prefix}### and ###{suffix}###',
+            f'"{match}": between ###{prefix}### and ###{suffix}###', prefix, match, suffix,
         )
 
     def __add_message_space_before(self, item, prefix, match, suffix):
-        item.add_error(
-            self.name,
+        self.add_warning('error', item,
             f"There should be a non-breakable space before "
             f'"{match[1:]}": between ###{prefix}{match[0]}### and '
-            f"###{match[1:]}{suffix}###",
+            f"###{match[1:]}{suffix}###", prefix, match, suffix,
         )

padpo/pofile.py

@@ -2,6 +2,7 @@
 
 import re
 from typing import List
+from polib import pofile
 
 import simplelogging
 
@@ -11,62 +12,31 @@
 class PoItem:
     """Translation item."""
 
-    def __init__(self, path, lineno):
+    def __init__(self, entry):
         """Initializer."""
-        self.path = path[3:]
-        self.lineno_start = lineno
-        self.lineno_end = lineno
-        self.parsing_msgid = None
-        self.msgid = []
-        self.msgstr = []
-        self.fuzzy = False
         self.warnings = []
         self.inside_pull_request = False
+        self.entry = entry
 
-    def append_line(self, line):
-        """Append a line of a `*.po` file to the item."""
-        self.lineno_end += 1
-        if line.startswith("msgid"):
-            self.parsing_msgid = True
-            self.msgid.append(line[7:-2])
-        elif line.startswith("msgstr"):
-            self.parsing_msgid = False
-            self.msgstr.append(line[8:-2])
-        elif line.startswith("#, fuzzy"):
-            self.fuzzy = True
-        elif line.startswith('"'):
-            if self.parsing_msgid:
-                self.msgid.append(line[1:-2])
-            elif not self.parsing_msgid is None:
-                self.msgstr.append(line[1:-2])
 
     def __str__(self):
         """Return string representation."""
         return (
-            f"    - {self.msgid_full_content}\n"
-            f"        => {self.msgstr_full_content}\n"
+            f"    - {self.entry.msgid}\n"
+            f"        => {self.entry.msgstr}\n"
             f"        => {self.msgstr_rst2txt}\n"
         )
 
-    @property
-    def msgid_full_content(self):
-        """Full content of the msgid."""
-        return "".join(self.msgid)
-
-    @property
-    def msgstr_full_content(self):
-        """Full content of the msgstr."""
-        return "".join(self.msgstr)
 
     @property
     def msgid_rst2txt(self):
         """Full content of the msgid (reStructuredText escaped)."""
-        return self.rst2txt(self.msgid_full_content)
+        return self.rst2txt(self.entry.msgid)
 
     @property
     def msgstr_rst2txt(self):
         """Full content of the msgstr (reStructuredText escaped)."""
-        return self.rst2txt(self.msgstr_full_content)
+        return self.rst2txt(self.entry.msgstr)
 
     @staticmethod
     def rst2txt(text):
@@ -112,18 +82,19 @@ def __init__(self, path=None):
     def parse_file(self, path):
         """Parse a `*.po` file according to its path."""
         # TODO assert path is a file, not a dir
-        item = None
-        with open(path, encoding="utf8") as f:
-            for lineno, line in enumerate(f):
-                if line.startswith("#: "):
-                    if item:
-                        self.content.append(item)
-                    item = PoItem(line, lineno + 1)
-                elif item:
-                    item.append_line(line)
-        if item:
+        self.pofile = pofile(path)
+        for entry in self.pofile:
+            item = PoItem(entry)
             self.content.append(item)
 
+        # lineno_end only needed for github patch. May be removed ?
+        import sys
+        lineno_end = sys.maxsize
+        for item in sorted(self.content, key=lambda x: x.entry.linenum, reverse=True):
+            item.lineno_end = lineno_end
+            lineno_end = item.entry.linenum - 1
+
+
     def __str__(self):
         """Return string representation."""
         ret = f"Po file: {self.path}\n"
@@ -148,7 +119,7 @@ def display_warnings(self, pull_request_info=None):
                         message.text,
                         extra={
                             "pofile": self.path,
-                            "poline": item.lineno_start,
+                            "poline": item.entry.linenum,
                             "checker": message.checker_name,
                             "leveldesc": "error",
                         },
@@ -159,7 +130,7 @@ def display_warnings(self, pull_request_info=None):
                         message.text,
                         extra={
                             "pofile": self.path,
-                            "poline": item.lineno_start,
+                            "poline": item.entry.linenum,
                             "checker": message.checker_name,
                             "leveldesc": "warning",
                         },
@@ -178,7 +149,7 @@ def tag_in_pull_request(self, pull_request_info):
                 item.inside_pull_request = False
             for lineno_diff in self.lines_in_diff(diff):
                 for item in self.content:
-                    if item.lineno_start <= lineno_diff <= item.lineno_end:
+                    if item.entry.linenum <= lineno_diff <= item.lineno_end:
                         item.inside_pull_request = True
 
     @staticmethod

pyproject.toml

@@ -24,6 +24,7 @@ python = "^3.7"
 pygrammalecte = "^1.3.0"
 requests = "^2.20.0"
 simplelogging = ">=0.10,<0.12"
+polib = "^1.1.1"
 
 [tool.poetry.dev-dependencies]
 python-dev-tools = {version = ">=2020.9.4", python = ">=3.7,<4.0"}

tests/po_without_warnings/abc.po

@@ -1,4 +1,4 @@
 #: ../Doc/library/typing.rst:594
 msgid "An ABC with one abstract method ``__int__``."
 msgstr ""
-msgstr "Une ABC avec une méthode abstraite ``__int__``."
+"Une ABC avec une méthode abstraite ``__int__``."

tests/po_without_warnings/silent_glossary.po

@@ -0,0 +1,14 @@
+# example how to silent glossary warning
+# first entry: 'statement' is correctly translated by a word found in the glossary
+#              --> nothing to do
+# second entry: 'tuple' is translated by a word not corresponding in the glossary, but that we know correct
+#               --> we silent padpo
+
+#: glossary.rst:25
+msgid "statement"
+msgstr "instruction"
+
+# silent-glossary:`tuple`
+#: glossary.rst:39
+msgid "tuple"
+msgstr "couple"

tests/po_without_warnings/silent_nbsp.po

@@ -0,0 +1,13 @@
+# example how to silent nbsp warnings
+# first entry is ok, with a nbsp before question mark
+# second entry raises a warning because of the walrus operator analysed as a colon
+# we silent padpo warning with a silent-nbsp-after tag
+
+#: test-nsbsp.rst:25
+msgid "What a question?"
+msgstr "Quelle question ?"
+
+# silent-nbsp-after:`morse`
+#: test-nbsp.rst:39
+msgid "the walrus operator `:=` is a new feature"
+msgstr "l'opérateur morse `:=` est fantastique"