Add documentation for the rest of the internals

Add documentation for most of the rest of the internals. Everything
should be mostly straightforwards, except for a few new places where I
had to forward some docstrings.

Signed-off-by: Sean Anderson <[email protected]>

doc/internals.rst

@@ -6,3 +6,39 @@ _mpmetrics
 
 .. automodule:: _mpmetrics
         :members:
+
+mpmetrics.atomic
+----------------
+
+.. automodule:: mpmetrics.atomic
+        :members:
+        :inherited-members:
+
+mpmetrics.generics
+------------------
+
+.. automodule:: mpmetrics.generics
+        :members:
+
+mpmetrics.heap
+--------------
+
+.. automodule:: mpmetrics.heap
+        :members:
+        :special-members: __init__
+
+mpmetrics.types
+---------------
+
+.. automodule:: mpmetrics.types
+        :members:
+        :special-members: __init__
+
+        .. autodata:: mpmetrics.types.Array
+        .. autodata:: mpmetrics.types.Box
+
+mpmetrics.util
+--------------
+
+.. automodule:: mpmetrics.util
+        :members:

doc/mpmetrics.rst

@@ -35,8 +35,16 @@ mpmetrics.metrics
 -----------------
 
 .. automodule:: mpmetrics.metrics
-        :members:
-        :special-members: __call__
+
+        .. autoclass:: mpmetrics.metrics.CollectorFactory
+                :members:
+                :special-members: __call__
+
+        .. autoclass:: mpmetrics.metrics.Collector
+                :members:
+
+        .. autoclass:: mpmetrics.metrics.LabeledCollector
+                :members:
 
 mpmetrics.flask
 ---------------

mpmetrics/atomic.py

@@ -1,6 +1,28 @@
 # SPDX-License-Identifier: LGPL-3.0-only
 # Copyright (C) 2022 Sean Anderson <[email protected]>
 
+"""
+multiprocess-safe atomics
+
+This module contains atomic types which automatically fall back to locking
+implementations on architectures which only support 32-bit atomics.
+
+.. py:data:: AtomicDouble
+
+    Either :py:class:`_mpmetrics.AtomicDouble`, or :py:class:`LockingDouble` if
+    the former is not supported.
+
+.. py:data:: AtomicInt64
+
+    Either :py:class:`_mpmetrics.AtomicInt64`, or :py:class:`LockingInt64` if
+    the former is not supported.
+
+.. py:data:: AtomicUInt64
+
+    Either :py:class:`_mpmetrics.AtomicUInt64`, or :py:class:`LockingUInt64` if
+    the former is not supported.
+"""
+
 import _mpmetrics
 from .types import Double, Int64, UInt64, Struct
 
@@ -12,14 +34,24 @@ class _Locking(Struct):
     }
 
     def get(self):
+        """Return the current value of the backing atomic"""
         with self._lock:
             return self._value.value
 
     def set(self, value):
+        """Set the backing atomic to `value`."""
         with self._lock:
             self._value.value = value
 
     def add(self, amount, raise_on_overflow=True):
+        """Add 'amount' to the backing atomic.
+
+        :param Union[int, float] amount: The amount to add
+        :param bool raise_on_overflow: Whether to raise an exception on overflow
+        :return: The value from before the addition.
+        :rtype: Union[int, float]
+        """
+
         with self._lock:
             old = self._value.value
             self._value.value = old + amount
@@ -28,16 +60,22 @@ def add(self, amount, raise_on_overflow=True):
             return old
 
 class LockingDouble(_Locking):
+    """An atomic double implemented using a lock"""
+
     _fields_ = _Locking._fields_ | {
         '_value': Double,
     }
 
 class LockingInt64(_Locking):
+    """An atomic 64-bit signed integer implemented using a lock"""
+
     _fields_ = _Locking._fields_ | {
         '_value': Int64,
     }
 
 class LockingUInt64(_Locking):
+    """An atomic 64-bit unsigned integer implemented using a lock"""
+
     _fields_ = _Locking._fields_ | {
         '_value': UInt64,
     }

mpmetrics/generics.py

@@ -1,17 +1,41 @@
 # SPDX-License-Identifier: LGPL-3.0-only
 # Copyright (C) 2022 Sean Anderson <[email protected]>
 
+"""Helpers for pickling polymorphic classes."""
+
 import importlib
 import itertools
 
 def saveattr(get):
+    """Save the result of `__getattr__`.
+
+    :param get: A `__getattr__` implementation
+
+    Wrap `get` and save the result with `setattr`.
+    """
     def wrapped(self, name):
         attr = get(self, name)
         setattr(self, name, attr)
         return attr
     return wrapped
 
 class ObjectType:
+    """Helper for classes polymorphic over classes.
+
+    This is a helper class to allow classes which are polymorphic over python
+    classes to be pickled. For example::
+
+        import pickle
+        from mpmetrics.generics import ObjectType
+
+        def MyClass(__name__, cls):
+            return type(__name__, (), locals())
+
+        MyClass = ObjectType('MyClass', MyClass)
+        assert MyClass[int].cls is int
+        assert pickle.loads(pickle.dumps(MyClass[int]())).cls is int
+    """
+
     class Attr:
         def __init__(self, name, cls, obj, nesting=1):
             self.name = name
@@ -50,6 +74,7 @@ def __getattr__(self, name):
 
     def __init__(self, name, cls):
         self.__qualname__ = name
+        self.__doc__ = cls.__doc__
         setattr(self, '<', self.Module(name + '.<', cls))
 
     def __getitem__(self, cls):
@@ -59,8 +84,25 @@ def __getitem__(self, cls):
         return getattr(parent, '>')
 
 class IntType:
+    """Helper for classes polymorphic over integers.
+
+    This is a helper class to allow classes which are polymorphic over ints
+    to be pickled. For example::
+
+        import pickle
+        from mpmetrics.generics import IntType
+
+        def MyClass(__name__, x):
+            return type(__name__, (), locals())
+
+        MyClass = IntType('MyClass', MyClass)
+        assert MyClass[5].x == 5
+        assert pickle.loads(pickle.dumps(MyClass[5]())).x == 5
+    """
+
     def __init__(self, name, cls):
         self.__qualname__ = name
+        self.__doc__ = cls.__doc__
         self.name = name
         self.cls = cls
 
@@ -72,8 +114,25 @@ def __getitem__(self, n):
         return getattr(self, repr(n))
 
 class FloatType:
+    """Helper for classes polymorphic over floats.
+
+    This is a helper class to allow classes which are polymorphic over floats
+    to be pickled. For example::
+
+        import pickle
+        from mpmetrics.generics import FloatType
+
+        def MyClass(__name__, x):
+            return type(__name__, (), locals())
+
+        MyClass = FloatType('MyClass', MyClass)
+        assert MyClass[2.7].x == 2.7
+        assert pickle.loads(pickle.dumps(MyClass[2.7]())).x == 2.7
+    """
+
     def __init__(self, name, cls):
         self.__qualname__ = name
+        self.__doc__ = cls.__doc__
         self.name = name
         self.cls = cls
 
@@ -86,8 +145,26 @@ def __getitem__(self, n):
 
 
 class ProductType:
+    """Helper to combine other types.
+
+    This is a helper class to allow classes which are polymorphic over multiple
+    types to be pickled. For example::
+
+        import pickle
+        from mpmetrics.generics import IntType, ObjectType, ProductType
+
+        def MyClass(__name__, cls, x):
+            return type(__name__, (), locals())
+
+        MyClass = ProductType('MyClass', MyClass, (ObjectType, IntType))
+        assert MyClass[int, 5].cls is int
+        assert MyClass[int, 5].x == 5
+        assert pickle.loads(pickle.dumps(MyClass[int, 5]())).x == 5
+    """
+
     def __init__(self, name, cls, argtypes, args=()):
         self.__qualname__ = name
+        self.__doc__ = cls.__doc__
         self.name = name
         self.cls = cls
         self.argtype = argtypes[0](self.name, self._chain)
@@ -113,8 +190,25 @@ def __getitem__(self, args):
         return argtype
 
 class ListType:
+    """Helper to combine other types.
+
+    This is a helper class to allow classes which are polymorphic over multiple
+    types to be pickled. For example::
+
+        import pickle
+        from mpmetrics.generics import IntType, ListType
+
+        def MyClass(__name__, xs):
+            return type(__name__, (), locals())
+
+        MyClass = ListType('MyClass', MyClass, IntType)
+        assert MyClass[1, 2, 3].xs == (1, 2, 3)
+        assert pickle.loads(pickle.dumps(MyClass[1, 2, 3]())).xs == (1, 2, 3)
+    """
+
     def __init__(self, name, cls, elemtype):
         self.__qualname__ = name
+        self.__doc__ = cls.__doc__
         self.name = name
         self.cls = cls
         self.elemtype = elemtype

mpmetrics/heap.py

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: LGPL-3.0-only
 # Copyright (C) 2021-22 Sean Anderson <[email protected]>
 
+"""A shared memory allocator."""
+
 import itertools
 import mmap
 from multiprocessing.reduction import DupFd
@@ -21,12 +23,36 @@
 PAGESIZE = 4096
 
 class Heap(Struct):
+    """A shared memory allocator.
+
+    This is a basic arena-style allocator. The core algorithm is (effectively)::
+
+        def malloc(size):
+            old_base = base
+            base += size
+            return old_base
+
+    We do not keep track of free blocks, so :py:meth:`Heap.Block.free` is a no-op.
+
+    Memory is requested from the OS in page-sized blocks. As we don't map all
+    of our memory up front, it's possible that different processes will map new
+    pages at different addresses. Therefore, we keep track of the address where
+    each page is mapped, and ensure blocks do not cross page boundaries.
+    Larger-than-page-size blocks are supported by aligning the block to the
+    page size and mapping all pages in that block in one go.
+    """
+
     _fields_ = {
         '_shared_lock': _mpmetrics.Lock,
         '_base': Size_t,
     }
 
     def __init__(self, map_size=PAGESIZE):
+        """Create a new Heap.
+
+        :param int map_size: The granularity to use when requesting memory from the OS
+        """
+
         if map_size % mmap.ALLOCATIONGRANULARITY:
             raise ValueError("size must be a multiple of {}".format(mmap.ALLOCATIONGRANULARITY))
         _align_check(map_size)
@@ -62,12 +88,29 @@ def __setstate__(self, state):
         super()._setstate(memoryview(self._maps[0])[:self.size])
 
     class Block:
+        """A block of memory allocated from a Heap."""
+
         def __init__(self, heap, start, size):
+            """Create a new Block.
+
+            :param Heap heap: The heap this block is from
+            :param int start: The offset of this block within the heap
+            :param int size: The size of this block
+            """
+
             self.heap = heap
             self.start = start
             self.size = size
 
         def deref(self):
+            """Dereference this block
+
+            :return: The memory referenced by this block
+            :rtype: memoryview
+
+            Dereference the block, faulting in unmapped pages as necessary.
+            """
+
             heap = self.heap
             first_page = int(self.start / heap.map_size)
             last_page = int((self.start + self.size - 1) / heap.map_size)
@@ -85,9 +128,21 @@ def deref(self):
             return memoryview(map)[off:off+self.size]
 
         def free(self):
+            """Free this block"""
             pass
 
     def malloc(self, size, alignment=CACHELINESIZE):
+        """Allocate shared memory.
+
+        :param int size: The amount of shared memory to allocate, in bytes
+        :param int alignment: The minimum alignment of the memory
+        :return: A block of shared memory
+        :rtype: Block
+
+        Allocate at least `size` bytes of shared memory. It will be aligned to
+        at least `alignment`.
+        """
+
         if size <= 0:
             raise ValueError("size must be strictly positive")
         elif size > self.map_size: