Add autogenerated docs for Python API (#129)

Automatically generate docs for docstub (the Python package) with
autosummary, including source links to the GitHub repository. For
now the entire API is private but it might still be a useful reference
for a potential plugin system.

This isn't perfect yet. For example, it seems that click commands
are somehow not picked up by autosummary. But that's something
for later.

Author: lagru

.gitignore

@@ -1,5 +1,6 @@
 # Project specific folders
 stubs/
+docs/python_api/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

docs/conf.py

@@ -1,10 +1,22 @@
 # Configuration file for the Sphinx documentation builder.
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
+# ruff: noqa: T201
+
+import inspect
+import shutil
+import sys
 from datetime import date
+from functools import cache
+from pathlib import Path
+from subprocess import check_output
 
 import setuptools_scm
 
+import docstub
+from docstub._path_utils import walk_source_package
+from docstub._utils import module_name_from_path
+
 # -- Project information ------------------------------------------------------
 
 project = "docstub"
@@ -22,14 +34,19 @@
 
 extensions = [
     "sphinx.ext.intersphinx",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.linkcode",
     "sphinx_copybutton",
-    # https://numpydoc.readthedocs.io/
     "myst_parser",
+    "numpydoc",
 ]
 
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3/", None),
     "typing": ("https://typing.python.org/en/latest/", None),
+    "lark": ("https://lark-parser.readthedocs.io/en/latest/", None),
+    "libcst": ("https://libcst.readthedocs.io/en/latest/", None),
 }
 
 myst_enable_extensions = [
@@ -41,6 +58,79 @@
 
 myst_heading_anchors = 3
 
+autosummary_generate = True
+
+autodoc_default_options = {"show-inheritance": True, "private-members": True}
+
+autodoc_member_order = "alphabetical"
+
+numpydoc_show_class_members = False
+
+add_module_names = False
+
+
+@cache
+def get_git_revision_hash():
+    """Extract hash of current revision."""
+    output = check_output(["git", "rev-parse", "HEAD"])
+    full_hash = output.decode().strip()
+    return full_hash
+
+
+def linkcode_resolve(domain, info):
+    """Resolve link for code objects.
+
+    This function is a hook called by the sphinx extension linkcode [1]_.
+    Loosely inspired by a function in NumPy [2]_.
+
+    Parameters
+    ----------
+    domain : str
+        Specifies the language domain of the given object.
+    info : dict
+        A dictionary describing the object.
+
+    Returns
+    -------
+    url : str
+        An URL to the source code.
+
+    References
+    ---------
+    .. [1] http://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html
+    .. [2] https://github.com/numpy/numpy/blob/cbf3a081271a43e980e3c2f76625deb43fd53922/doc/source/conf.py#L303-L357
+    """
+    if domain != "py" or "module" not in info or "fullname" not in info:
+        return None
+
+    # Get reference to the object, start with the module
+    obj = sys.modules.get(info["module"])
+    for part in info["fullname"].split("."):
+        try:
+            obj = getattr(obj, part)
+        except AttributeError:
+            # Not all attributes exist, e.g. instance attributes
+            return None
+    # Strip decorators which would resolve to the source of the decorator
+    obj = inspect.unwrap(obj)
+
+    file = info["module"].replace(".", "/")
+    try:
+        source, start_line = inspect.getsourcelines(obj)
+    except (TypeError, OSError):
+        # Skip unexpected types, occurs when `obj` isn't a module, class, method,
+        # function, etc.
+        return None
+    stop_line = start_line + len(source) - 1
+
+    template = "https://github.com/scientific-python/docstub/blob/{revision}/src/{file}.py#L{start}-L{stop}"
+    revision = get_git_revision_hash()
+    assert len(revision) == 40
+    url = template.format(
+        revision=revision, file=file, start=start_line, stop=stop_line
+    )
+    return url
+
 
 # -- HTML output --------------------------------------------------------------
 
@@ -79,3 +169,112 @@
         "sidebar/variant-selector.html",
     ]
 }
+
+
+# -- Pregenerate doc sources ---------------------------------------------------
+
+PYTHON_API_TEMPLATE = """.. Generated by `conf.py`
+
+Python API
+==========
+
+Docstub is primarily intended to be used as a command line tool.
+Nevertheless we document it's Python facing API here.
+
+Modules
+-------
+
+.. autosummary::
+  :toctree:
+  :recursive:
+  :template: module.rst
+  :nosignatures:
+
+"""
+
+
+def is_private_qualname(name):
+    parts = name.split(".")
+    return any(p.startswith("_") for p in parts)
+
+
+def write_python_api(module, *, exclude_modules=()):
+    """Create ``python_api/index.rst`` as an entrypoint for autosummary.
+
+    Parameters
+    ----------
+    module : ModuleType
+    exclude_modules : tuple[str, ...]
+    """
+    package_path = Path(module.__file__).parent
+
+    names = sorted(
+        module_name_from_path(path) for path in walk_source_package(package_path)
+    )
+    names = [n for n in names if n not in exclude_modules]
+
+    # Sort private modules after public ones
+    public_names = [n for n in names if not is_private_qualname(n)]
+    private_names = [n for n in names if is_private_qualname(n)]
+    names = public_names + private_names
+
+    names = [f"  {name}" for name in names]
+    content = "\n".join([PYTHON_API_TEMPLATE, *names])
+
+    api_dir = Path(__file__).parent / "python_api"
+    if api_dir.exists():
+        shutil.rmtree(api_dir)
+        print(f"[conf.py] deleted {api_dir}")
+
+    api_dir.mkdir()
+    index_path = api_dir / "index.rst"
+    index_path.write_text(content)
+    print(f"[conf.py] wrote {index_path}")
+
+
+write_python_api(
+    docstub,
+    exclude_modules=("docstub.__main__", "docstub._version"),
+)
+
+
+# -- Patch Sphinx --------------------------------------------------------------
+
+
+def autodoc_skip_members(app, obj_type, name, obj, skip, options):
+    """Force autosummary to show private members too."""
+    # TODO ensure that click commands are included as well
+    if name.startswith("_") and not name.startswith("__"):
+        # Force autosummary to show private members to,
+        # `autodoc_default_options = {"private-members": True}`
+        # doesn't solve this on its own
+        return False
+    return None  # Default behavior
+
+
+def truncate_signature_defaults(
+    app, what, name, obj, options, signature, return_annotation
+):
+    """Truncate long defaults in function signatures."""
+    if what in {"module", "property", "attribute"}:
+        return signature, return_annotation
+
+    try:
+        signature_object = inspect.signature(obj)
+    except (TypeError, ValueError):
+        pass
+    else:
+        for param in signature_object.parameters.values():
+            if param.default is inspect.Parameter.empty:
+                continue
+            default = repr(param.default)
+            if len(default) > 30:
+                truncated = default[:12] + "..." + default[-12:]
+                signature = signature.replace(default, truncated)
+
+    return signature, return_annotation
+
+
+def setup(app):
+    app.connect("autodoc-process-signature", truncate_signature_defaults)
+    app.connect("autodoc-skip-member", autodoc_skip_members)

docs/index.md

@@ -32,6 +32,7 @@ development/index
 command_line
 configuration
 typing_syntax
+python_api/index
 glossary
 release_notes/index
 :::

docs/static/furo_overrides.css

@@ -33,4 +33,17 @@ article .align-default {
 .sidebar-brand-text {
     margin-bottom: 0;
 
-}
+}
+
+/* Use more readable sizes and spacing for headings */
+h2, .h2 {
+    font-size: 1.5em;
+    margin-top: 3rem;
+}
+h3, .h3 {
+    font-size: 1.25em;
+    margin-top: 1rem;
+}
+h4, .h4 {
+    font-size: 1.1em;
+}

docs/templates/module.rst

@@ -0,0 +1,43 @@
+``{{ fullname }}``
+=={{ underline }}==
+
+.. currentmodule:: {{ fullname }}
+
+.. automodule:: {{ fullname }}
+
+{% block functions %}
+{% if functions %}
+{{ _('Functions') | underline(line='-') }}
+
+.. autosummary::
+    :toctree:
+    :template: objects.rst
+    :nosignatures:
+
+  {% for item in functions if not item.startswith("_") %}
+    {{ item -}}
+  {% endfor %}
+  {% for item in functions if item.startswith("_") %}
+    {{ item -}}
+  {% endfor %}
+  {% endif %}
+  {%- endblock %}
+
+{% block classes %}
+{% if classes %}
+{{ _('Classes') | underline(line='-') }}
+
+.. autosummary::
+    :toctree:
+    :recursive:
+    :template: objects.rst
+    :nosignatures:
+
+  {% for item in classes if not item.startswith("_") %}
+    {{ item -}}
+  {% endfor %}
+  {% for item in classes if item.startswith("_") %}
+    {{ item -}}
+  {% endfor %}
+{% endif %}
+{%- endblock %}

docs/templates/objects.rst

@@ -0,0 +1,15 @@
+``{{ name }}``
+=={{ underline }}==
+
+in module :mod:`{{ module }}`.
+
+.. currentmodule:: {{ module }}
+
+{% if objtype == "class" -%}
+.. autoclass:: {{ fullname }}
+  :members:
+  :private-members:
+  :special-members:
+{% else %}
+.. auto{{ objtype }}:: {{ fullname }}
+{% endif %}

src/docstub-stubs/_config.pyi

@@ -12,6 +12,7 @@ logger: logging.Logger
 
 @dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
 class Config:
+
     TEMPLATE_PATH: ClassVar[Path]
     NUMPY_PATH: ClassVar[Path]
 

src/docstub/__init__.py

@@ -1,6 +1,12 @@
 # Copyright (c) 2024 Lars Grüter. All rights reserved.
 
-"""Generate Python stub files (PYI) from docstrings."""
+"""Generate Python stub files (PYI) from docstrings.
+
+Attributes
+----------
+__version__ : str
+    The current version.
+"""
 
 from ._version import __version__
 

src/docstub/_analysis.py

@@ -1,4 +1,4 @@
-"""Collect type information."""
+"""Collection of type information."""
 
 import builtins
 import importlib

src/docstub/_app_generate_stubs.py

@@ -1,6 +1,7 @@
-"""Implementation of `docstub run`.
+"""Implementation of ``docstub run``.
 
-Its interface declaration is in `_cli.py`.
+Its interface declaration is in ``_cli.py``. Splitting declaration and
+implementation (with slow imports) speeds up the ``--help`` option.
 """
 
 import logging
@@ -88,8 +89,8 @@ def _collect_type_info(root_path, *, ignore=(), cache=False):
     ignore : Sequence[str], optional
         Don't yield files matching these glob-like patterns. The pattern is
         interpreted relative to the root of the Python package unless it starts
-        with "/". See :ref:`glob.translate(..., recursive=True, include_hidden=True)`
-        for more details on the precise implementation.
+        with "/". See :func:`glob.translate` with recursion and hidden files
+        enabled for more details on the precise implementation.
     cache : bool, optional
         Cache collected types.
 
@@ -140,6 +141,7 @@ def _collect_type_info(root_path, *, ignore=(), cache=False):
 
 @contextmanager
 def _log_execution_time():
+    """Measure and log execution time of wrapped function or block."""
     start = time.time()
     try:
         yield
@@ -255,7 +257,8 @@ def generate_stubs(
     no_cache,
     verbosity,
 ):
-    """
+    """Generate Python stub files (implementation of ``docstub run``).
+
     Parameters
     ----------
     root_path : Path