improve dalapi parser by warning undocumented instance [issue-1162]

docs/dalapi/doxypy/__init__.py

@@ -19,3 +19,4 @@
 from .loader import TransformerPass, NameTransformer
 from .listing import ListingReader
 from . import model
+from .filter import UndocumentedWarnFilter

docs/dalapi/doxypy/filter.py

@@ -0,0 +1,83 @@
+# file: filter.py
+#===============================================================================
+# Copyright 2019-2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+
+import re
+from typing import (
+    List,
+    Tuple,
+    Union,
+    Dict,
+    Text,
+    NewType
+)
+from .parser import UndocumentedWarn
+
+IgnoreConfigT = NewType("IgnoreConfigT", Tuple[Text, Union[Text, Dict[Text, Text]]])
+
+class UndocumentedWarnFilter:
+
+    PROP_MAPPING = {
+        'Compound': {
+            'identifier': 'member'
+        },
+        'Param': {
+            'function': 'parent'
+        }
+    }
+
+    MAIN_PROP = {
+        'Param': 'parent',
+        'Compound': 'member',
+        'General': 'parent'
+    }
+
+    def __init__(self, ignore_configs: List[IgnoreConfigT]):
+        self._ignore_filters: Dict[Text, List[Dict[Text, re.Pattern]]] = {
+            'Compound': [],
+            'Param': [],
+            'General': []
+        }
+        self._init_ignore_filters(ignore_configs)
+
+    def ignored(self, warn: UndocumentedWarn) -> bool:
+        for filters in self._ignore_filters[warn.type_name]:
+            for prop, pattern in filters.items():
+                if not pattern.fullmatch(getattr(warn, prop)):
+                    break
+            else:
+                return True
+        return False
+
+    def _init_ignore_filters(self, ignore_configs: List[IgnoreConfigT]):
+        for type_name, config in ignore_configs:
+            if type_name not in self._ignore_filters:
+                raise ValueError(f"invalid ignore config type_name: {type_name}")
+            if isinstance(config, str):
+                prop = self.MAIN_PROP[type_name]
+                self._ignore_filters[type_name].append({prop: re.compile(config)})
+            elif isinstance(config, dict):
+                self._ignore_filters[type_name].append({
+                    self._map_prop(type_name, prop): re.compile(regex)
+                        for prop, regex in config.items()
+                })
+            else:
+                raise ValueError(f"unknown config: {config}, type: {type(config)}")
+
+    @classmethod
+    def _map_prop(cls, type_name, prop):
+        return cls.PROP_MAPPING.get(type_name, {}).get(prop, prop)

docs/dalapi/doxypy/index.py

@@ -52,6 +52,7 @@ def __init__(self, parser: Parser, loader: ModelLoader,
         self._loader = loader
         self._name_transformer = name_transformer
         self._index = self._initialize()
+        self._warns = []
 
     def find(self, query: str):
         try:
@@ -75,6 +76,17 @@ def _initialize(self):
                     name = self._name_transformer.transform(entry.name)
                 yield name, model_object
 
+    @utils.return_dict
+    def _initialize_warns(self):
+        for kind, warns in self._parser.parse_warning().items():
+            yield kind, list(warns)
+
+    @property
+    def warns(self):
+        if len(self._warns) <= 0:
+            self._warns = self._initialize_warns()
+        return self._warns
+
     def _find_inner(self, query):
         parent_name, name = utils.split_compound_name(query)
         model = self._index[parent_name].model

docs/dalapi/doxypy/parser/__init__.py

@@ -25,6 +25,12 @@
 )
 from .index import parse as parse_index
 from .compound import parse as parse_compound
+from .warning_log import parse as parse_warning
+from .warning_log import (
+    ParsedWarn,
+    UndocumentedWarn,
+)
+
 
 class Parser(object):
     def __init__(self, xml_dir: Text):
@@ -35,6 +41,12 @@ def parse(self, refid):
         parse = parse_index if refid == 'index' else parse_compound
         return parse(xml_filename, silence=True)
 
+    def parse_warning(self):
+        warning_log_path = os.path.join(os.path.dirname(os.path.dirname(self._dir)), 'warning.log')
+        if not os.path.exists(warning_log_path):
+            return {}
+        return parse_warning(warning_log_path)
+
     def _resolve_path(self, refid):
         xml_name = os.path.join(self._dir, refid)
         return f'{xml_name}.xml'

docs/dalapi/doxypy/parser/warning_log.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+# file: warning_log.py
+#===============================================================================
+# Copyright 2019-2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+# -*- coding: utf-8 -*-
+
+from typing import (
+    List,
+    Dict,
+)
+import re
+
+
+class RawWarn:
+    def __init__(self, filepath, lineno, text):
+        self.filepath = filepath
+        self.lineno = lineno
+        self.text = text
+        self.subitems = []
+
+    def add_subitem(self, subitem):
+        self.subitems.append(subitem)
+
+    def __str__(self):
+        return f'{self.filepath}:{self.lineno}: {self.text} {self.subitems if self.subitems else ""}'
+
+
+class ParsedWarn:
+    def __init__(self, warn: RawWarn):
+        self._filepath = warn.filepath
+        self._lineno = warn.lineno
+        self._raw = warn
+        self._parse(warn)
+
+    def __init_subclass__(cls, **kwargs):
+        cls.__wname__ = cls.__name__[:-4]
+
+    def __str__(self):
+        return str(self._raw)
+
+    def _parse(self, warn: RawWarn):
+        raise NotImplementedError
+
+    @staticmethod
+    def is_instance(warn: RawWarn):
+        raise NotImplementedError
+
+    @property
+    def filepath(self):
+        return self._filepath
+
+
+class UnknownWarn(ParsedWarn):
+    def _parse(self, warn: RawWarn):
+        pass
+
+    @staticmethod
+    def is_instance(warn: RawWarn):
+        return True
+
+
+class UndocumentedWarn(ParsedWarn):
+
+    TYPE_GENERAL = 0
+    TYPE_PARAM = 1
+    TYPE_COMPOUND = 2
+
+    TYPE_NAMES = ['General', 'Param', 'Compound']
+
+    normal_regex = re.compile(r'^ warning: Member (?P<member>.+?) \((?P<member_type>\w+)\) of (?P<parent_type>\w+) (?P<parent>.+?) is not documented\.$')
+    compound_regex = re.compile(r'^ warning: Compound (?P<member>.+?) is not documented\.$')
+    param_regex = re.compile(r'^ warning: The following parameter of (?P<parent>.+?) is not documented:$')
+    parameter_regex = re.compile(r"^parameter '(?P<parameter>[^']+')$")
+
+    @property
+    def type_name(self):
+        return self.TYPE_NAMES[self.type]
+
+    def _parse(self, warn: RawWarn):
+        self.type = self.TYPE_GENERAL
+        self.member = None
+        self.member_type = None
+        self.parent = None
+        self.parent_type = None
+        self.parameters = []
+
+        if warn.text[-1] == ':':  # param
+            self.type = self.TYPE_PARAM
+            match = self.param_regex.match(warn.text)
+            if not match:
+                raise RuntimeError(f"Unknown UndocumentedWarn Format!: {warn}")
+            self.parent = match.group('parent')
+            self.parameter = [self.__extract_parameter(item, warn) for item in warn.subitems]
+        elif warn.text[10:18] == 'Compound':
+            self.type = self.TYPE_COMPOUND
+            match = self.compound_regex.match(warn.text)
+            if not match:
+                raise RuntimeError(f"Unknown UndocumentedWarn Format!: {warn}")
+            self.member = match.group('member')
+        else:
+            match = self.normal_regex.match(warn.text)
+            if not match:
+                raise RuntimeError(f"Unknown UndocumentedWarn Format!: {warn}")
+            self.member = match.group('member')
+            self.member_type = match.group('member_type')
+            self.parent = match.group('parent')
+            self.parent_type = match.group('parent_type')
+
+    @classmethod
+    def __extract_parameter(cls, item, warn):
+        match = cls.parameter_regex.match(item)
+        if not match:
+            raise RuntimeError(f"Unknown UndocumentedWarn Paramter Format!: {item} of {warn}")
+        return match.group('parameter')
+
+    @staticmethod
+    def is_instance(warn: RawWarn):
+        return 'not documented' == warn.text[-15:-1]
+
+
+WarnKlasses = [
+    UndocumentedWarn,
+    UnknownWarn
+]
+
+
+def parse_log(in_file) -> List[RawWarn]:
+    warnings = []
+    with open(in_file) as f:
+        for line in f.readlines():
+            items = line.strip().split(':', 2)
+            if len(items) == 1:
+                warnings[-1].add_subitem(line.strip())
+            elif len(items) == 3:
+                warnings.append(RawWarn(*items))
+            else:
+                raise RuntimeError(f"Unknown warning format! warning line: {line}")
+
+    return warnings
+
+
+def parse_warn(raw_warns: List[RawWarn]) -> Dict[str, List[ParsedWarn]]:
+    parsed_warns = {warn_klass.__wname__: [] for warn_klass in WarnKlasses}
+    for warn in raw_warns:
+        for warn_klass in WarnKlasses:
+            if not warn_klass.is_instance(warn):
+                continue
+            parsed_warns[warn_klass.__wname__].append(warn_klass(warn))
+            break
+
+    return parsed_warns
+
+
+def parse(in_file):
+    return parse_warn(parse_log(in_file))

docs/dalapi/extension.py

@@ -18,6 +18,7 @@
 import os
 import time
 from typing import (Dict, Tuple, Text)
+import sphinx.util.logging
 from . import doxypy
 from . import utils
 from . import roles
@@ -110,15 +111,22 @@ def __init__(self, app):
         self._listing = None
         self._path_resolver = None
         self._is_listing_enabled = False
+        self._is_warning_undocumented_enabled = False
+        self._info_ignored_undocumented = False
+        self._undocumented_warn_filter = doxypy.UndocumentedWarnFilter([])
         self._read_env()
 
-    def configure(self, relative_doxyfile_dir, relative_sources_dir, is_listing_enabled):
+    def configure(self, relative_doxyfile_dir, relative_sources_dir, is_listing_enabled,
+                  is_warning_undocumented_enabled, info_ignored_undocumented, ignored_undocumented_warnings):
         self._path_resolver = PathResolver(
             self.app,
             relative_doxyfile_dir,
             relative_sources_dir
         )
         self._is_listing_enabled = is_listing_enabled
+        self._is_warning_undocumented_enabled = is_warning_undocumented_enabled
+        self._info_ignored_undocumented = info_ignored_undocumented
+        self._undocumented_warn_filter = doxypy.UndocumentedWarnFilter(ignored_undocumented_warnings)
 
     @property
     def current_docname(self):
@@ -134,6 +142,9 @@ def index(self) -> doxypy.Index:
                     transformers.RstDescriptionTransformer(),
                 ]
             )
+            if self._is_warning_undocumented_enabled:
+                self._log_undocumented_warns()
+
         return self._index
 
     @property
@@ -169,6 +180,15 @@ def get_env_flag(env_var):
         self.debug = get_env_flag('DALAPI_DEBUG')
         self.always_rebuild = get_env_flag('DALAPI_ALWAYS_REBUILD')
 
+    def _log_undocumented_warns(self):
+        logger = sphinx.util.logging.getLogger('DoxygenWarns')
+        for warn in self._index.warns.get('Undocumented', []):
+            if self._undocumented_warn_filter.ignored(warn):
+                if self._info_ignored_undocumented:
+                    logger.info('[Undocumented]%s', warn)
+                continue
+            logger.warning(warn)
+
 
 class EventHandler(object):
     def __init__(self, ctx: Context):
@@ -181,7 +201,10 @@ def get_config_values(self, app):
         self.ctx.configure(
             relative_doxyfile_dir=app.config.onedal_relative_doxyfile_dir,
             relative_sources_dir=app.config.onedal_relative_sources_dir,
-            is_listing_enabled=app.config.onedal_enable_listing
+            is_listing_enabled=app.config.onedal_enable_listing,
+            is_warning_undocumented_enabled=app.config.onedal_enable_warning_undocumented,
+            ignored_undocumented_warnings=app.config.onedal_ignored_undocumented_warnings,
+            info_ignored_undocumented=app.config.onedal_info_ignored_undocumented
         )
 
 def setup(app):
@@ -199,6 +222,9 @@ def setup(app):
     app.add_config_value('onedal_relative_doxyfile_dir', '.', 'env')
     app.add_config_value('onedal_relative_sources_dir', '.', 'env')
     app.add_config_value('onedal_enable_listing', True, 'env')
+    app.add_config_value('onedal_enable_warning_undocumented', False, 'env')
+    app.add_config_value('onedal_info_ignored_undocumented', False, 'env')
+    app.add_config_value('onedal_ignored_undocumented_warnings', [], 'env')
 
     handler = EventHandler(ctx)
     app.connect("builder-inited", handler.get_config_values)