From bb4d3a38ae55479d3f8b7212676a0765e60fcf1e Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Wed, 22 Nov 2023 16:45:26 -0500 Subject: [PATCH] Add documentation for the rest of the internals Add documentation for most of the rest of the internals. Everything should be mostly straightforwards, except for a few new places where I had to forward some docstrings. Signed-off-by: Sean Anderson --- doc/internals.rst | 36 ++++++++ doc/mpmetrics.rst | 12 ++- mpmetrics/atomic.py | 38 ++++++++ mpmetrics/generics.py | 94 +++++++++++++++++++ mpmetrics/heap.py | 55 +++++++++++ mpmetrics/metrics.py | 28 +++++- mpmetrics/types.py | 206 ++++++++++++++++++++++++++++++++++++++++-- mpmetrics/util.py | 35 +++++++ 8 files changed, 492 insertions(+), 12 deletions(-) diff --git a/doc/internals.rst b/doc/internals.rst index f39530d..8b6c4aa 100644 --- a/doc/internals.rst +++ b/doc/internals.rst @@ -6,3 +6,39 @@ _mpmetrics .. automodule:: _mpmetrics :members: + +mpmetrics.atomic +---------------- + +.. automodule:: mpmetrics.atomic + :members: + :inherited-members: + +mpmetrics.generics +------------------ + +.. automodule:: mpmetrics.generics + :members: + +mpmetrics.heap +-------------- + +.. automodule:: mpmetrics.heap + :members: + :special-members: __init__ + +mpmetrics.types +--------------- + +.. automodule:: mpmetrics.types + :members: + :special-members: __init__ + + .. autodata:: mpmetrics.types.Array + .. autodata:: mpmetrics.types.Box + +mpmetrics.util +-------------- + +.. automodule:: mpmetrics.util + :members: diff --git a/doc/mpmetrics.rst b/doc/mpmetrics.rst index 776368c..7f6d10f 100644 --- a/doc/mpmetrics.rst +++ b/doc/mpmetrics.rst @@ -35,8 +35,16 @@ mpmetrics.metrics ----------------- .. automodule:: mpmetrics.metrics - :members: - :special-members: __call__ + + .. autoclass:: mpmetrics.metrics.CollectorFactory + :members: + :special-members: __call__ + + .. autoclass:: mpmetrics.metrics.Collector + :members: + + .. autoclass:: mpmetrics.metrics.LabeledCollector + :members: mpmetrics.flask --------------- diff --git a/mpmetrics/atomic.py b/mpmetrics/atomic.py index eda27e5..ea0fab2 100644 --- a/mpmetrics/atomic.py +++ b/mpmetrics/atomic.py @@ -1,6 +1,28 @@ # SPDX-License-Identifier: LGPL-3.0-only # Copyright (C) 2022 Sean Anderson +""" +multiprocess-safe atomics + +This module contains atomic types which automatically fall back to locking +implementations on architectures which only support 32-bit atomics. + +.. py:data:: AtomicDouble + + Either :py:class:`_mpmetrics.AtomicDouble`, or :py:class:`LockingDouble` if + the former is not supported. + +.. py:data:: AtomicInt64 + + Either :py:class:`_mpmetrics.AtomicInt64`, or :py:class:`LockingInt64` if + the former is not supported. + +.. py:data:: AtomicUInt64 + + Either :py:class:`_mpmetrics.AtomicUInt64`, or :py:class:`LockingUInt64` if + the former is not supported. +""" + import _mpmetrics from .types import Double, Int64, UInt64, Struct @@ -12,14 +34,24 @@ class _Locking(Struct): } def get(self): + """Return the current value of the backing atomic""" with self._lock: return self._value.value def set(self, value): + """Set the backing atomic to `value`.""" with self._lock: self._value.value = value def add(self, amount, raise_on_overflow=True): + """Add 'amount' to the backing atomic. + + :param Union[int, float] amount: The amount to add + :param bool raise_on_overflow: Whether to raise an exception on overflow + :return: The value from before the addition. + :rtype: Union[int, float] + """ + with self._lock: old = self._value.value self._value.value = old + amount @@ -28,16 +60,22 @@ def add(self, amount, raise_on_overflow=True): return old class LockingDouble(_Locking): + """An atomic double implemented using a lock""" + _fields_ = _Locking._fields_ | { '_value': Double, } class LockingInt64(_Locking): + """An atomic 64-bit signed integer implemented using a lock""" + _fields_ = _Locking._fields_ | { '_value': Int64, } class LockingUInt64(_Locking): + """An atomic 64-bit unsigned integer implemented using a lock""" + _fields_ = _Locking._fields_ | { '_value': UInt64, } diff --git a/mpmetrics/generics.py b/mpmetrics/generics.py index 3881534..f2ff23f 100644 --- a/mpmetrics/generics.py +++ b/mpmetrics/generics.py @@ -1,10 +1,18 @@ # SPDX-License-Identifier: LGPL-3.0-only # Copyright (C) 2022 Sean Anderson +"""Helpers for pickling polymorphic classes.""" + import importlib import itertools def saveattr(get): + """Save the result of `__getattr__`. + + :param get: A `__getattr__` implementation + + Wrap `get` and save the result with `setattr`. + """ def wrapped(self, name): attr = get(self, name) setattr(self, name, attr) @@ -12,6 +20,22 @@ def wrapped(self, name): return wrapped class ObjectType: + """Helper for classes polymorphic over classes. + + This is a helper class to allow classes which are polymorphic over python + classes to be pickled. For example:: + + import pickle + from mpmetrics.generics import ObjectType + + def MyClass(__name__, cls): + return type(__name__, (), locals()) + + MyClass = ObjectType('MyClass', MyClass) + assert MyClass[int].cls is int + assert pickle.loads(pickle.dumps(MyClass[int]())).cls is int + """ + class Attr: def __init__(self, name, cls, obj, nesting=1): self.name = name @@ -50,6 +74,7 @@ def __getattr__(self, name): def __init__(self, name, cls): self.__qualname__ = name + self.__doc__ = cls.__doc__ setattr(self, '<', self.Module(name + '.<', cls)) def __getitem__(self, cls): @@ -59,8 +84,25 @@ def __getitem__(self, cls): return getattr(parent, '>') class IntType: + """Helper for classes polymorphic over integers. + + This is a helper class to allow classes which are polymorphic over ints + to be pickled. For example:: + + import pickle + from mpmetrics.generics import IntType + + def MyClass(__name__, x): + return type(__name__, (), locals()) + + MyClass = IntType('MyClass', MyClass) + assert MyClass[5].x == 5 + assert pickle.loads(pickle.dumps(MyClass[5]())).x == 5 + """ + def __init__(self, name, cls): self.__qualname__ = name + self.__doc__ = cls.__doc__ self.name = name self.cls = cls @@ -72,8 +114,25 @@ def __getitem__(self, n): return getattr(self, repr(n)) class FloatType: + """Helper for classes polymorphic over floats. + + This is a helper class to allow classes which are polymorphic over floats + to be pickled. For example:: + + import pickle + from mpmetrics.generics import FloatType + + def MyClass(__name__, x): + return type(__name__, (), locals()) + + MyClass = FloatType('MyClass', MyClass) + assert MyClass[2.7].x == 2.7 + assert pickle.loads(pickle.dumps(MyClass[2.7]())).x == 2.7 + """ + def __init__(self, name, cls): self.__qualname__ = name + self.__doc__ = cls.__doc__ self.name = name self.cls = cls @@ -86,8 +145,26 @@ def __getitem__(self, n): class ProductType: + """Helper to combine other types. + + This is a helper class to allow classes which are polymorphic over multiple + types to be pickled. For example:: + + import pickle + from mpmetrics.generics import IntType, ObjectType, ProductType + + def MyClass(__name__, cls, x): + return type(__name__, (), locals()) + + MyClass = ProductType('MyClass', MyClass, (ObjectType, IntType)) + assert MyClass[int, 5].cls is int + assert MyClass[int, 5].x == 5 + assert pickle.loads(pickle.dumps(MyClass[int, 5]())).x == 5 + """ + def __init__(self, name, cls, argtypes, args=()): self.__qualname__ = name + self.__doc__ = cls.__doc__ self.name = name self.cls = cls self.argtype = argtypes[0](self.name, self._chain) @@ -113,8 +190,25 @@ def __getitem__(self, args): return argtype class ListType: + """Helper to combine other types. + + This is a helper class to allow classes which are polymorphic over multiple + types to be pickled. For example:: + + import pickle + from mpmetrics.generics import IntType, ListType + + def MyClass(__name__, xs): + return type(__name__, (), locals()) + + MyClass = ListType('MyClass', MyClass, IntType) + assert MyClass[1, 2, 3].xs == (1, 2, 3) + assert pickle.loads(pickle.dumps(MyClass[1, 2, 3]())).xs == (1, 2, 3) + """ + def __init__(self, name, cls, elemtype): self.__qualname__ = name + self.__doc__ = cls.__doc__ self.name = name self.cls = cls self.elemtype = elemtype diff --git a/mpmetrics/heap.py b/mpmetrics/heap.py index 1baa9cb..f4b5c8e 100644 --- a/mpmetrics/heap.py +++ b/mpmetrics/heap.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: LGPL-3.0-only # Copyright (C) 2021-22 Sean Anderson +"""A shared memory allocator.""" + import itertools import mmap from multiprocessing.reduction import DupFd @@ -21,12 +23,36 @@ PAGESIZE = 4096 class Heap(Struct): + """A shared memory allocator. + + This is a basic arena-style allocator. The core algorithm is (effectively):: + + def malloc(size): + old_base = base + base += size + return old_base + + We do not keep track of free blocks, so :py:meth:`Heap.Block.free` is a no-op. + + Memory is requested from the OS in page-sized blocks. As we don't map all + of our memory up front, it's possible that different processes will map new + pages at different addresses. Therefore, we keep track of the address where + each page is mapped, and ensure blocks do not cross page boundaries. + Larger-than-page-size blocks are supported by aligning the block to the + page size and mapping all pages in that block in one go. + """ + _fields_ = { '_shared_lock': _mpmetrics.Lock, '_base': Size_t, } def __init__(self, map_size=PAGESIZE): + """Create a new Heap. + + :param int map_size: The granularity to use when requesting memory from the OS + """ + if map_size % mmap.ALLOCATIONGRANULARITY: raise ValueError("size must be a multiple of {}".format(mmap.ALLOCATIONGRANULARITY)) _align_check(map_size) @@ -62,12 +88,29 @@ def __setstate__(self, state): super()._setstate(memoryview(self._maps[0])[:self.size]) class Block: + """A block of memory allocated from a Heap.""" + def __init__(self, heap, start, size): + """Create a new Block. + + :param Heap heap: The heap this block is from + :param int start: The offset of this block within the heap + :param int size: The size of this block + """ + self.heap = heap self.start = start self.size = size def deref(self): + """Dereference this block + + :return: The memory referenced by this block + :rtype: memoryview + + Dereference the block, faulting in unmapped pages as necessary. + """ + heap = self.heap first_page = int(self.start / heap.map_size) last_page = int((self.start + self.size - 1) / heap.map_size) @@ -85,9 +128,21 @@ def deref(self): return memoryview(map)[off:off+self.size] def free(self): + """Free this block""" pass def malloc(self, size, alignment=CACHELINESIZE): + """Allocate shared memory. + + :param int size: The amount of shared memory to allocate, in bytes + :param int alignment: The minimum alignment of the memory + :return: A block of shared memory + :rtype: Block + + Allocate at least `size` bytes of shared memory. It will be aligned to + at least `alignment`. + """ + if size <= 0: raise ValueError("size must be strictly positive") elif size > self.map_size: diff --git a/mpmetrics/metrics.py b/mpmetrics/metrics.py index d00a6f4..6b9fca7 100644 --- a/mpmetrics/metrics.py +++ b/mpmetrics/metrics.py @@ -70,9 +70,21 @@ def _family(self): return metrics_core.Metric(self._name, self._docs, self._metric._typ) def describe(self): + """Describe the metric + + :return: An iterator yielding one metric with no samples + :rtype: Iterator[prometheus_client.metrics_core.Metric] + """ + yield self._family() def collect(self): + """Collect samples from the metric + + :return: An iterator yielding one metric with collected samples. + :rtype: Iterator[prometheus_client.metrics_core.Metric] + """ + family = self._family() def add_sample(suffix, value, labels={}, exemplar=None): family.add_sample(self._name + suffix, labels, value, exemplar=exemplar) @@ -187,9 +199,21 @@ def _family(self): return metrics_core.Metric(self._name, self._docs, self._metric._typ) def describe(self): + """Describe the metric + + :return: An iterator yielding one metric with no samples + :rtype: Iterator[prometheus_client.metrics_core.Metric] + """ + yield self._family() def collect(self): + """Collect samples from the metric + + :return: An iterator yielding one metric with samples collected from each label. + :rtype: Iterator[prometheus_client.metrics_core.Metric] + """ + family = self._family() with self._lock: with self._shared_lock: @@ -241,8 +265,8 @@ def __call__(self, name, documentation, labelnames=(), namespace="", :param str subsystem: A subsystem name for the metric. This will be prepended to `name` after `namespace`. :param str unit: The unit of measurement for the metric. This will be appended to `name`. - :param registry: The registry to register this metric with. It will collect data from the - metric. + :param prometheus_client.registry.CollectorRegistry registry: The registry to register this + metric with. It will collect data from the metric. :param \\**kwargs: Any additional arguments are passed to the metric itself. :return: A new metric :rtype: Option[Collector, LabeledCollector] diff --git a/mpmetrics/types.py b/mpmetrics/types.py index 27ce0e1..c97e209 100644 --- a/mpmetrics/types.py +++ b/mpmetrics/types.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: LGPL-3.0-only # Copyright (C) 2022 Sean Anderson +"""Various types backed by (shared) memory""" + import ctypes import io from multiprocessing.reduction import ForkingPickler @@ -10,15 +12,50 @@ from .generics import IntType, ObjectType, ProductType from .util import align, classproperty -def _wrap_ctype(__name__, ctype): +def _wrap_ctype(__name__, ctype, doc): size = ctypes.sizeof(ctype) align = ctypes.alignment(ctype) + __doc__ = f"""{doc.capitalize()} backed by (shared) memory. + + .. py:attribute:: value + :type: {'float' if __name__ == 'Double' else 'int'} + + The value itself. You can read and modify this value as necessary. For + example:: + + from mpmetrics.heap import Heap + from mpmetrics.types import Box, {__name__} + + var = Box[{__name__}](Heap()) + assert var.value == 0 + var.value += 1 + assert var.value == 1 + + .. py:attribute:: size + :type: int + :value: {size} + + The size of {doc}, in bytes + + .. py:attribute:: align + :type: int + :value: {align} + + The alignment of {doc}, in bytes + """ + def __init__(self, mem, heap=None): self._mem = mem self._value = ctype.from_buffer(mem) self._value.value = 0 + __init__.__doc__ = f"""Create a new {__name__}. + + :param memoryview mem: The backing memory + :param heap: Unused + """ + def _setstate(self, mem, heap=None, **kwargs): self._mem = mem self._value = ctype.from_buffer(mem) @@ -35,15 +72,46 @@ def __setattr__(self, name, value): def __delattr__(self, name): delattr(self.__dict__['_value'], name) - return type(__name__, (), { name: value for name, value in locals().items() - if name != 'ctype'}) + ns = locals() + del ns['doc'] + del ns['ctype'] + return type(__name__, (), ns) -Double = _wrap_ctype('Double', ctypes.c_double) -Size_t = _wrap_ctype('Size_t', ctypes.c_size_t) -Int64 = _wrap_ctype('Int64', ctypes.c_int64) -UInt64 = _wrap_ctype('UInt64', ctypes.c_uint64) +Double = _wrap_ctype('Double', ctypes.c_double, "a double") +Size_t = _wrap_ctype('Size_t', ctypes.c_size_t, "a size_t") +Int64 = _wrap_ctype('Int64', ctypes.c_int64, "an int64_t") +UInt64 = _wrap_ctype('UInt64', ctypes.c_uint64, "a uint64_t") class Struct: + """A structured group of fields backed by (shared) memory. + + This is a base class that can be subclassed to create C-style structs:: + + from mpmetrics.heap import Heap + from mpmetrics.types import Double, Size_t, Struct + + class MyStruct(mpmetrics.types.Struct): + _fields_ = { + 'a': Double, + 'b': Size_t, + } + + assert MyStruct.size == Double.size + Size_t.size + s = Box[MyStruct](Heap()) + assert type(s.a) == Double + assert type(s.b) == Size_t + + .. py:property:: _fields_ + :classmethod: + :type: dict[str, Any] + + The fields of the struct, in order. Upon initialization, each value is + initialized with a block of memory equal to its `.size`. Padding is + added as necessary to ensure alignment. + + Subclasses must implement this property. + """ + @classmethod def _fields_iter(cls): off = 0 @@ -54,15 +122,23 @@ def _fields_iter(cls): @classproperty def size(cls): + """The size of the struct, in bytes""" for name, field, off in cls._fields_iter(): size = field.size + off return size @classproperty def align(cls): + """The alignment of the struct, in bytes""" return max(field.align for field in cls._fields_.values()) def __init__(self, mem, heap=None): + """Create a new Struct. + + :param memoryview mem: The backing memory + :param mpmetrics.heap.Heap heap: Passed to each field's ``__init__`` + """ + self._mem = mem for name, field, off in self._fields_iter(): setattr(self, name, field(mem[off:off + field.size], heap=heap)) @@ -75,6 +151,27 @@ def _setstate(self, mem, heap=None, **kwargs): setattr(self, name, field) def Array(__name__, cls, n): + """An array of values backed by (shared) memory. + + You can access values in an `Array` just like it was a `list`:: + + from mpmetrics.heap import Heap + from mpmetrics.types import Array, Box, Double + + assert Array[Double, 5].size == Double.size * 5 + a = Box[Array[Double, 5]](Heap()) + assert type(a[0]) == Double + a[4].value = 6.28 + + + .. py:method:: Array.__init__(mem, heap=None) + + Create a new Array. + + :param memoryview mem: The backing memory + :param mpmetrics.heap.Heap heap: Passed to each member's ``__init__`` + """ + if n < 1: raise ValueError("n must be strictly positive") @@ -120,6 +217,61 @@ def __iter__(self): Array = ProductType('Array', Array, (ObjectType, IntType)) class _Box: + """A heap-allocated box to put values in + + This class "boxes" another class using heap-allocated memory. For example, + you could create a `Double` like:: + + from mpmetrics.heap import Heap + from mpmetrics.types import Double + + block = Heap().malloc(Double.size) + d = Double(block.deref()) + + But d._mem is a `memoryview` which can't be pickled. `Box` takes care of + keeping track of the memory block:: + + from mpmetrics.heap import Heap + from mpmetrics.types import Box, Double + + d = Box[Double](Heap()) + + .. py:method:: Box.__init__(heap, *args, **kwargs) + + Create a new object on the heap + + :param mpmetrics.heap.Heap heap: The heap to use when allocating the object + :param \\*args: Any additional arguments are passed to the boxed class + :param \\**kwargs: Any additional keyword arguments are passed to the boxed class. + + The superclass's `__init__` is called with a newly-allocated buffer as + the first argument, any positional arguments to this function, the + keyword argument `heap` set to `heap`, and any additional keyword + arguments to this function. + + .. py:method:: Box._getstate() + :abstractmethod: + + Return keyword arguments to pass to `_setstate`. + + :return: A dictionary of keyword arguments for `_setstate` + :rtype: dict + + This method is optional; if it is not implemented then no additional + keyword arguments will be passed to `_setstate`. + + .. py:method:: Box._setstate(mem, heap=None, **kwargs) + :abstractmethod: + + Initialize internal state after unpickling. + + :param memoryview mem: The backing memory + :param mpmetrics.heap.Heap heap: The heap `mem` was allocated from + :param \**kwargs: Any additional arguments from `_getstate` + + This method must be implemented by boxed types. + """ + def __init__(self, heap, *args, **kwargs): block = heap.malloc(self.size) super().__init__(block.deref(), *args, heap=heap, **kwargs) @@ -136,7 +288,11 @@ def __setstate__(self, state): self.__block, kwargs = state super()._setstate(self.__block.deref(), heap=self.__block.heap, **kwargs) -Box = ObjectType('Box', lambda name, cls: type(name, (_Box, cls), {'__doc__': cls.__doc__})) +def _create_box(name, cls): + return type(name, (_Box, cls), {'__doc__': cls.__doc__}) + +_create_box.__doc__ = _Box.__doc__ +Box = ObjectType('Box', _create_box) class _Pickler(ForkingPickler): def __init__(self, file, heap, protocol=None): @@ -169,6 +325,22 @@ def loads(cls, data, heap): return cls(buf, heap).load() class Object(Struct): + """A python object pickled in (shared) memory + + This is a base class for python objects backed by shared memory. Whenever + the object is accessed, it is unpickled from the backing memory. When it is + modified, it is pickled to the backing memory. + + This class itself does not contain the actual object. Instead, it contains + the start/size/length of the block containing the object. When the object + grows too large for the block, the old block is free'd and a new one is + allocated. + + This class provides no synchronization. All methods should be accessed + under some other form of synchonization, such as a + :py:class:`_mpmetrics.Lock`. + """ + _fields_ = { '_start': Size_t, '_size': Size_t, @@ -176,6 +348,12 @@ class Object(Struct): } def __init__(self, mem, heap): + """Create a new Object. + + :param memoryview mem: The memory used to store information about the buffer + :param mpmetrics.heap.Heap heap: The heap to use when (re)allocating the buffer + """ + if not heap: raise ValueError("heap must be provided") super().__init__(mem) @@ -337,6 +515,12 @@ def update(self, other=()): return self._mutate('update', other) class Dict(Object, Sequence, MutableMapping): + """A `dict` backed by (shared) memory. + + All methods of `dict` are supported. External synchronization (such as from + a :py:class:`_mpmetrics.Lock`) must be provided when accessing any method. + """ + _new = dict def __or__(self, other): @@ -350,6 +534,12 @@ def copy(self): return self._object class List(Object, MutableSequence): + """A `list` backed by (shared) memory. + + All methods of `list` are supported. External synchronization (such as from + a :py:class:`_mpmetrics.Lock`) must be provided when accessing any method. + """ + _new = list def sort(self, key=None, reverse=False): diff --git a/mpmetrics/util.py b/mpmetrics/util.py index baba89e..11453b6 100644 --- a/mpmetrics/util.py +++ b/mpmetrics/util.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: LGPL-3.0-only # Copyright (C) 2022 Sean Anderson +"""Various small utilities.""" + def _align_mask(x, mask): return (x + mask) & ~mask @@ -9,17 +11,50 @@ def _align_check(a): raise ValueError("{} is not a power of 2".format(a)) def align(x, a): + """Align `x` to `a` + + :param int x: The value to align + :param int a: The alignment; must be a power of two + :return: The smallest multiple of `a` greater than `x` + :rtype: int + """ + _align_check(a) return _align_mask(x, a - 1) def align_down(x, a): + """Align `x` down to `a` + + :param int x: The value to align + :param int a: The alignment; must be a power of two + :return: The largest multiple of `a` less than `x` + :rtype: int + """ + _align_check(a) return x & ~(a - 1) def genmask(hi, lo): + """Generate a mask with bits between `hi` `lo` set. + + :param int hi: The highest bit to set, inclusive + :param int lo: The lowest bit to set, inclusive + :return: The bitmask + :rtype: int + + `hi` must be greater than `lo`. Bits are numbered in "little-endian" order, + starting from zero. The following invariant holds:: + + mask = 0 + for n in range(lo, hi): + mask |= 1 << n + assert mask == genmask(hi, lo) + """ + return (-1 << lo) & ~(-1 << (hi + 1)) # From https://stackoverflow.com/a/7864317/5086505 class classproperty(property): + """Like `property` but for classes.""" def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)()