Merge pull request #98 from piotrhyzy/feature/group_unhandled_paths

[Feature] Add `group_unhandled_paths` Option to Enhance `filter_unhandled_paths` Functionality  #91

README.md

@@ -82,6 +82,8 @@ retrieves a value from the `Request` object. [See below](#labels) for examples.
 
 `filter_unhandled_paths`: setting this to `True` will cause the middleware to ignore requests with unhandled paths (in other words, 404 errors). This helps prevent filling up the metrics with 404 errors and/or intentially bad requests. Default is `True`.
 
+`group_unhandled_paths`: Similar to `filter_unhandled_paths`, but instead of ignoring the requests, they are grouped under the `__unknown__` path. This option overrides `filter_unhandled_paths` by setting it to `False`. The default value is `False`.
+
 `buckets`: accepts an optional list of numbers to use as histogram buckets. The default value is `None`, which will cause the library to fall back on the Prometheus defaults (currently `[0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0]`).
 
 `skip_paths`: accepts an optional list of paths, or regular expressions for paths, that will not collect metrics. The default value is `None`, which will cause the library to collect metrics on every requested path. This option is useful to avoid collecting metrics on health check, readiness or liveness probe endpoints.

starlette_exporter/middleware.py

@@ -3,6 +3,7 @@
 import logging
 import re
 import time
+import warnings
 from collections import OrderedDict
 from contextlib import suppress
 from inspect import iscoroutine
@@ -95,11 +96,22 @@ def __init__(
         always_use_int_status: bool = False,
         labels: Optional[Mapping[str, Union[str, Callable]]] = None,
         exemplars: Optional[Callable] = None,
+        group_unhandled_paths: bool = False,
     ):
         self.app = app
         self.app_name = app_name
         self.prefix = prefix
         self.group_paths = group_paths
+
+        if group_unhandled_paths and filter_unhandled_paths:
+            filter_unhandled_paths = False
+            warnings.warn(
+                "filter_unhandled_paths was set to True but has been changed to False "
+                "because group_unhandled_paths is True and these settings are mutually exclusive",
+                UserWarning,
+            )
+
+        self.group_unhandled_paths = group_unhandled_paths
         self.filter_unhandled_paths = filter_unhandled_paths
 
         self.kwargs = {}
@@ -407,7 +419,7 @@ async def wrapped_send(message: Message) -> None:
             else:
                 status_code = 500
 
-        if self.filter_unhandled_paths or self.group_paths:
+        if self.filter_unhandled_paths or self.group_paths or self.group_unhandled_paths:
             grouped_path: Optional[str] = None
 
             endpoint = scope.get("endpoint", None)
@@ -422,6 +434,10 @@ async def wrapped_send(message: Message) -> None:
                     raise exception
                 return
 
+            # group_unhandled_paths works similar to filter_unhandled_paths, but instead of
+            # removing the request from the metrics, it groups it under a single path.
+            if self.group_unhandled_paths and grouped_path is None:
+                path = "__unknown__"
 
             # group_paths enables returning the original router path (with url param names)
             # for example, when using this option, requests to /api/product/1 and /api/product/3

tests/test_middleware.py

@@ -162,6 +162,19 @@ def test_404_filter(self, client):
 
         assert "/404" not in metrics
 
+    def test_404_group_unhandled_paths_on(self, testapp):
+        """test that an unknown path is captured in metrics if group_unhandled_paths=True"""
+
+        client = TestClient(testapp(group_unhandled_paths=True, filter_unhandled_paths=False))
+        client.get("/404")
+
+        metrics = client.get("/metrics").content.decode()
+
+        assert (
+            """starlette_requests_total{app_name="starlette",method="GET",path="__unknown__",status_code="404"} 1.0"""
+            in metrics
+        )
+
     def test_unhandled(self, client):
         """test that an unhandled exception still gets logged in the requests counter"""