api: add metrics endpoint (#630)

Adds a metrics endpoint that computes updated metric values on each
request.

On each request we:
- fetch runner groups from Redis
- get all runners for each group
- update a Gauge for the runners_count on each group
- render out the prometheus metrics in standard format

Resolves #625

Author: lewismiddleton

poetry.lock

@@ -27,6 +27,7 @@ rq-scheduler = "^0.13.1"
 pyvmomi = "^8.0.2.0.1"
 vapi-runtime = { url = "https://raw.githubusercontent.com/vmware/vsphere-automation-sdk-python/v8.0.1.0/lib/vapi-runtime/vapi_runtime-2.40.0-py2.py3-none-any.whl" }
 vcenter-bindings = { url = "https://raw.githubusercontent.com/vmware/vsphere-automation-sdk-python/v8.0.1.0/lib/vcenter-bindings/vcenter_bindings-4.1.0-py2.py3-none-any.whl" }
+prometheus-client = "^0.20.0"
 
 
 [tool.poetry.group.docs]

pyproject.toml

@@ -8,7 +8,14 @@
 from runner_manager import Runner, RunnerGroup, Settings, log
 from runner_manager.dependencies import get_queue, get_redis, get_settings
 from runner_manager.jobs.startup import startup
-from runner_manager.routers import _health, private, public, runner_groups, webhook
+from runner_manager.routers import (
+    _health,
+    metrics,
+    private,
+    public,
+    runner_groups,
+    webhook,
+)
 
 settings = get_settings()
 log.setLevel(settings.log_level)
@@ -39,6 +46,7 @@ async def lifespan(app: FastAPI):
 app.include_router(private.router)
 app.include_router(public.router)
 app.include_router(runner_groups.router)
+app.include_router(metrics.router)
 
 
 @app.get("/")

runner_manager/main.py

@@ -0,0 +1,22 @@
+from typing import List
+
+from fastapi import APIRouter
+from fastapi.responses import PlainTextResponse
+from prometheus_client import Gauge, generate_latest
+
+from runner_manager import RunnerGroup
+from runner_manager.models.runner import Runner
+
+router = APIRouter(prefix="/metrics")
+
+runners_count = Gauge("runners_count", "Number of runners", ["runner_group"])
+
+
+@router.get("/", response_class=PlainTextResponse)
+def compute_metrics() -> PlainTextResponse:
+    groups: List[RunnerGroup] = RunnerGroup.find().all()
+    for group in groups:
+        runners: List[Runner] = group.get_runners()
+        runners_count.labels(runner_group=group.name).set(len(runners))
+    metrics = generate_latest().decode()
+    return PlainTextResponse(content=metrics)

runner_manager/routers/metrics.py

@@ -0,0 +1,65 @@
+from fastapi.testclient import TestClient
+
+from runner_manager import RunnerGroup
+from runner_manager.clients.github import GitHub
+
+
+def test_metrics_endpoint(client: TestClient, runner_group: RunnerGroup):
+    runner_group.save()
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    runner_lines = [
+        line for line in response.text.splitlines() if line.startswith("runners_")
+    ]
+    assert f'runners_count{{runner_group="{runner_group.name}"}} 0.0' in runner_lines
+
+
+def test_runner_count(client: TestClient, runner_group: RunnerGroup, github: GitHub):
+    runner_group.save()
+    want = len(runner_group.get_runners())
+
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    got = [line for line in response.text.splitlines() if line.startswith("runners_")]
+    print(want)
+    assert f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}' in got
+
+    runner_group.max = 2
+    runner_group.min = 1
+    runner_group.save()
+
+    want = 0.0
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    before = [
+        line for line in response.text.splitlines() if line.startswith("runners_")
+    ]
+    assert f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}' in before
+
+    runner = runner_group.create_runner(github)
+    assert runner is not None
+
+    want = 1.0
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    after_create = [
+        line for line in response.text.splitlines() if line.startswith("runners_")
+    ]
+    assert (
+        f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}'
+        in after_create
+    )
+
+    runner = runner_group.delete_runner(runner, github)
+    assert runner is not None
+
+    want = 0.0
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    after_delete = [
+        line for line in response.text.splitlines() if line.startswith("runners_")
+    ]
+    assert (
+        f'runners_count{{runner_group="{runner_group.name}"}} {want:.1f}'
+        in after_delete
+    )