feat(core): implement prometheus metrics

see #231

Author: mikonse

Diff for: abrechnung/application/groups.py

@@ -1,9 +1,9 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import asyncpg
 from sftkit.database import Connection
 from sftkit.error import InvalidArgument
-from sftkit.service import Service, with_db_transaction
+from sftkit.service import Service, with_db_connection, with_db_transaction
 
 from abrechnung.config import Config
 from abrechnung.core.auth import create_group_log
@@ -20,6 +20,7 @@
     GroupPreview,
 )
 from abrechnung.domain.users import User
+from abrechnung.util import timed_cache
 
 
 class GroupService(Service[Config]):
@@ -461,3 +462,8 @@ async def unarchive_group(self, *, conn: Connection, user: User, group_id: int):
             "update grp set archived = false where id = $1",
             group_id,
         )
+
+    @with_db_connection
+    @timed_cache(timedelta(minutes=5))
+    async def total_number_of_groups(self, conn: Connection):
+        return await conn.fetchval("select count(*) from grp")

Diff for: abrechnung/application/transactions.py

@@ -1,11 +1,11 @@
 import base64
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Optional, Union
 
 import asyncpg
 from sftkit.database import Connection
 from sftkit.error import InvalidArgument
-from sftkit.service import Service, with_db_transaction
+from sftkit.service import Service, with_db_connection, with_db_transaction
 
 from abrechnung.application.common import _get_or_create_tag_ids
 from abrechnung.config import Config
@@ -25,6 +25,7 @@
     UpdateTransaction,
 )
 from abrechnung.domain.users import User
+from abrechnung.util import timed_cache
 
 
 class TransactionService(Service[Config]):
@@ -721,3 +722,20 @@ async def _create_pending_transaction_change(
         )
 
         return revision_id
+
+    @with_db_connection
+    @timed_cache(timedelta(minutes=5))
+    async def total_number_of_transactions(self, conn: Connection):
+        return await conn.fetchval("select count(*) from transaction_state_valid_at(now()) where not deleted")
+
+    @with_db_connection
+    @timed_cache(timedelta(minutes=5))
+    async def total_amount_of_money_per_currency(self, conn: Connection) -> dict[str, float]:
+        result = await conn.fetch(
+            "select t.currency_symbol, sum(t.value) as total_value "
+            "from transaction_state_valid_at(now()) as t where not t.deleted group by t.currency_symbol"
+        )
+        aggregated = {}
+        for row in result:
+            aggregated[row["currency_symbol"]] = row["total_value"]
+        return aggregated

Diff for: abrechnung/config.py

@@ -67,6 +67,11 @@ class AuthConfig(BaseModel):
     auth: Optional[AuthConfig] = None
 
 
+class MetricsConfig(BaseModel):
+    enabled: bool = False
+    expose_money_amounts: bool = False
+
+
 class Config(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="ABRECHNUNG_", env_nested_delimiter="__")
 
@@ -77,6 +82,7 @@ class Config(BaseSettings):
     # in case all params are optional this is needed to make the whole section optional
     demo: DemoConfig = DemoConfig()
     registration: RegistrationConfig = RegistrationConfig()
+    metrics: MetricsConfig = MetricsConfig()
 
     @classmethod
     def settings_customise_sources(

Diff for: abrechnung/http/api.py

@@ -1,6 +1,7 @@
 import json
 import logging
 
+from prometheus_fastapi_instrumentator import Instrumentator
 from sftkit.http import Server
 
 from abrechnung import __version__
@@ -11,6 +12,7 @@
 from abrechnung.config import Config
 from abrechnung.database.migrations import check_revision_version, get_database
 
+from . import metrics
 from .context import Context
 from .routers import accounts, auth, common, groups, transactions
 
@@ -64,9 +66,20 @@ async def _setup(self):
     async def _teardown(self):
         await self.db_pool.close()
 
+    def _instrument_api(self):
+        if not self.cfg.metrics.enabled:
+            return
+        instrumentor = Instrumentator()
+        instrumentor.add(metrics.abrechnung_number_of_groups_total(self.group_service))
+        instrumentor.add(metrics.abrechnung_number_of_transactions_total(self.transaction_service))
+        if self.cfg.metrics.expose_money_amounts:
+            instrumentor.add(metrics.abrechnung_total_amount_of_money(self.transaction_service))
+        instrumentor.instrument(self.server.api).expose(self.server.api, endpoint="/api/metrics")
+
     async def run(self):
         await self._setup()
         try:
+            self._instrument_api()
             await self.server.run(self.context)
         finally:
             await self._teardown()

Diff for: abrechnung/http/metrics.py

@@ -0,0 +1,46 @@
+from prometheus_client import Gauge
+from prometheus_fastapi_instrumentator.metrics import Info
+
+from abrechnung.application.groups import GroupService
+from abrechnung.application.transactions import TransactionService
+
+
+def abrechnung_number_of_groups_total(group_service: GroupService):
+    metric = Gauge(
+        "abrechnung_number_of_groups_total",
+        "Number of groups created on this Abrechnung instance..",
+    )
+
+    async def instrumentation(info: Info) -> None:
+        total = await group_service.total_number_of_groups()
+        metric.set(total)
+
+    return instrumentation
+
+
+def abrechnung_number_of_transactions_total(transaction_service: TransactionService):
+    metric = Gauge(
+        "abrechnung_number_of_transactions_total",
+        "Number of transactions created on this Abrechnung instance..",
+    )
+
+    async def instrumentation(info: Info) -> None:
+        total = await transaction_service.total_number_of_transactions()
+        metric.set(total)
+
+    return instrumentation
+
+
+def abrechnung_total_amount_of_money(transaction_service: TransactionService):
+    metric = Gauge(
+        "abrechnung_total_amount_of_money",
+        "Total amount of money per currency cleared via thisthis Abrechnung instance..",
+        labelnames=("currency_symbol",),
+    )
+
+    async def instrumentation(info: Info) -> None:
+        total = await transaction_service.total_amount_of_money_per_currency()
+        for currency, value in total.items():
+            metric.labels(currency).set(value)
+
+    return instrumentation

Diff for: abrechnung/util.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import re
 import uuid
@@ -72,3 +73,23 @@ def is_valid_uuid(val: str):
         return True
     except ValueError:
         return False
+
+
+def timed_cache(ttl: timedelta):
+    def wrapper(func):
+        last_execution: datetime | None = None
+        last_value = None
+
+        @functools.wraps(func)
+        async def wrapped(*args, **kwargs):
+            nonlocal last_execution, last_value
+            current_execution = datetime.now()
+            if last_execution is None or last_value is None or current_execution - last_execution > ttl:
+                last_value = await func(*args, **kwargs)
+                last_execution = current_execution
+
+            return last_value
+
+        return wrapped
+
+    return wrapper

Diff for: docs/development/setup.rst

@@ -11,32 +11,28 @@ Setup
 Installation
 ------------
 
-Fork and clone the repository ::
+Fork and clone the repository
+
+.. code-block:: shell
 
   git clone https://github.com/SFTtech/abrechnung.git
   cd abrechnung
 
-Then install the package in local development mode as well as all required dependencies. Make sure to have
-`flit <https://github.com/pypa/flit>`_ installed first. Installing the dependencies can be two ways:
+Then install the package in local development mode as well as all required dependencies.
 
-Setup a virtual environment and install the packages via pip (straightforward way) ::
+Setup a virtual environment and install the packages via pip
 
-  virtualenv -p python3 venv
-  source venv/bin/activate
-  pip install flit
-  flit install -s --deps develop
+.. code-block:: shell
 
-Or install the dependencies through your package manager (useful for distribution packaging)
+  virtualenv -p python3 .venv
+  source .venv/bin/activate
+  pip install -e . '[dev,test]'
 
-* arch linux (slight chance some dependencies may be missing here)
+Additionally you probably will want to activate the git pre-commit hooks (for formatting and linting) by running
 
 .. code-block:: shell
 
-  sudo pacman -S python-flit python-yaml python-aiohttp python-aiohttp-cors python-asyncpg python-sphinx python-schema python-email-validator python-bcrypt python-pyjwt python-aiosmtpd python-pytest python-pytest-cov python-black python-mypy python-pylint python-apispec python-marshmallow python-webargs
-
-Afterwards install the package without dependencies ::
-
-  flit install -s --deps none
+  pre-commit install
 
 Database Setup
 --------------
@@ -60,7 +56,9 @@ Create the database (in a psql prompt):
 * Launch ``abrechnung -c abrechnung.yaml api``
 * Launch ``abrechnung -c abrechnung.yaml mailer`` to get mail delivery (working mail server in config file required!)
 
-It is always possible wipe and rebuild the database with ::
+It is always possible wipe and rebuild the database with
+
+.. code-block:: shell
 
   abrechnung -c abrechnung.yaml db rebuild
 
@@ -71,7 +69,7 @@ In case a new features requires changes to the database schema create a new migr
 
 .. code-block:: shell
 
-  ./tools/create_revision.py <revision_name>
+  sftkit create-migration <revision_name>
 
 In case you did not install the abrechnung in development mode it might be necessary to add the project root folder
 to your ``PYTHONPATH``.
@@ -100,30 +98,49 @@ is used as a means to wipe and repopulate the database between tests.
 
   alter schema public owner to "<your user>"
 
-Finally run the tests via ::
+Finally run the tests via
+
+.. code-block:: shell
 
   make test
 
-Run the linters via ::
+Run the linters via
+
+.. code-block:: shell
 
   make lint
 
+Run the formatters via
+
+.. code-block:: shell
+
+  make format
+
 Frontend Development
 --------------------
 
-Working on the frontend is quite easy, simply ::
+Working on the frontend is quite easy, simply
+
+.. code-block:: shell
 
   cd web
-  yarn install
-  yarn start
+  npm install
+  npx nx serve web
 
 and you are good to go!
 
 Documentation
 -------------
 
-To build the documentation locally simply run ::
+To build the documentation locally simply run
 
+.. code-block:: shell
+
+  pip install -r docs/requires.txt
   make docs
 
-The html docs can then be found in ``docs/_build``.
+The html docs can then be found in ``docs/_build`` or served locally with
+
+.. code-block:: shell
+
+  make serve-docs

Diff for: docs/usage/configuration.rst

@@ -15,11 +15,13 @@ Database
 The first step after installing the **abrechnung** is to setup the database. Due to the use of database specific features
 we only support **PostgreSQL** with versions >= 13. Other versions might work but are untested.
 
-First create a database with an associated user ::
+First create a database with an associated user
 
-  $ sudo -u postgres psql
-  > create user abrechnung with password '<some secure password>';
-  > create database abrechnung owner abrechnung;
+.. code-block:: shell
+
+  sudo -u postgres psql
+  create user abrechnung with password '<some secure password>';
+  create database abrechnung owner abrechnung;
 
 Enter the information into the config file in ``/etc/abrechnung/abrechnung.yaml`` under the section database as
 
@@ -31,7 +33,9 @@ Enter the information into the config file in ``/etc/abrechnung/abrechnung.yaml`
     dbname: "abrechnung"
     password: "<password>"
 
-Apply all database migrations with ::
+Apply all database migrations with
+
+.. code-block:: shell
 
   abrechnung db migrate
 
@@ -48,7 +52,9 @@ The ``name`` is used to populate the email subjects as ``[<name>] <subject>``.
 API Config
 ---------------
 Typically the config for the http API does not need to be changed much apart from two important settings!
-In the ``api`` section make sure to insert a newly generated secret key, e.g. with ::
+In the ``api`` section make sure to insert a newly generated secret key, e.g. with
+
+.. code-block:: shell
 
   pwgen -S 64 1
 
@@ -124,6 +130,30 @@ Guest users will not be able to create new groups themselves but can take part i
     valid_email_domains: ["some-domain.com"]
     allow_guest_users: true
 
+Prometheus Metrics
+------------------
+
+Abrechnung also provides prometheus metrics which are disabled by default.
+This includes some general metrics about the abrechnung instance such as
+
+- http request durations and groupings of error codes
+- general python environment metrics such as process utilization and garbage collection performance
+
+Additionally it currently includes the following set of abrechnung specific metrics
+
+- number of groups created on the instance
+- number of transactions created on the instance
+- total amount of money by currency which was cleared via the instance, i.e. the total sum of transaction values per currency over all groups.
+  This is disabled by default as it may expose private data on very small abrechnung instances.
+
+To enable metrics under the api endpoint ``/api/metrics`` simply add the following to the config file
+
+.. code-block:: yaml
+
+  metrics:
+    enabled: true
+    expose_money_amounts: false  # disabled by default
+
 Configuration via Environment Variables
 ---------------------------------------