Skip to content

Add custom labels support for metrics #7207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add custom labels support for metrics #7207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion caddyconfig/httpcaddyfile/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -472,8 +472,20 @@ func unmarshalCaddyfileMetricsOptions(d *caddyfile.Dispenser) (any, error) {
switch d.Val() {
case "per_host":
metrics.PerHost = true
case "labels":
if metrics.Labels == nil {
metrics.Labels = make(map[string]string)
}
for nesting := d.Nesting(); d.NextBlock(nesting); {
key := d.Val()
if !d.NextArg() {
return nil, d.ArgErr()
}
value := d.Val()
metrics.Labels[key] = value
}
default:
return nil, d.Errf("unrecognized servers option '%s'", d.Val())
return nil, d.Errf("unrecognized metrics option '%s'", d.Val())
}
}
return metrics, nil
Expand Down
60 changes: 59 additions & 1 deletion caddyconfig/httpcaddyfile/options_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,65 @@
}

if string(out) != tc.output {
t.Errorf("Test %d error output mismatch Expected: %s, got %s", i, tc.output, out)
t.Errorf("Test %d error output mismatch Expected: %s, got %s", i, tc.output, string(out))
}
}
}

func TestGlobalMetricsOptionSyntax(t *testing.T) {
for i, tc := range []struct {
input string
expectError bool
}{
{
input: `{
metrics {
per_host
}
}`,
expectError: false,
},
{
input: `{
metrics {
labels {
proto "{http.request.proto}"
method "{http.request.method}"
}
}
}`,
expectError: false,
},
{
input: `{
metrics {
per_host
labels {
proto "{http.request.proto}"
host "{http.request.host}"
}
}
}`,
expectError: false,
},
{
input: `{
metrics {
unknown_option
}
}`,
expectError: true,
},
} {
adapter := caddyfile.Adapter{
ServerType: ServerType{},
}

out, _, err := adapter.Adapt([]byte(tc.input), nil)

Check failure on line 115 in caddyconfig/httpcaddyfile/options_test.go

View workflow job for this annotation

GitHub Actions / test (linux, 1.25) 

declared and not used: out

Check failure on line 115 in caddyconfig/httpcaddyfile/options_test.go

View workflow job for this annotation

GitHub Actions / test (mac, 1.25) 

declared and not used: out

Check failure on line 115 in caddyconfig/httpcaddyfile/options_test.go

View workflow job for this annotation

GitHub Actions / test (windows, 1.25) 

declared and not used: out

if err != nil != tc.expectError {
t.Errorf("Test %d error expectation failed Expected: %v, got %v", i, tc.expectError, err)
continue
}
}
}
47 changes: 47 additions & 0 deletions caddytest/integration/caddyfile_adapt/custom_labels_metrics.caddyfiletest
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
metrics {
labels {
proto "{http.request.proto}"
method "{http.request.method}"
client_ip "{http.request.remote}"
host "{http.request.host}"
}
}
}

:8080 {
respond "Hello World" 200
}
----------
{
"apps": {
"http": {
"servers": {
"srv0": {
"listen": [
":8080"
],
"routes": [
{
"handle": [
{
"body": "Hello World",
"handler": "static_response",
"status_code": 200
}
]
}
]
}
},
"metrics": {
"labels": {
"client_ip": "{http.request.remote}",
"host": "{http.request.host}",
"method": "{http.request.method}",
"proto": "{http.request.proto}"
}
}
}
}
}
53 changes: 53 additions & 0 deletions modules/caddyhttp/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ type Metrics struct {
// managed by Caddy.
PerHost bool `json:"per_host,omitempty"`

// Labels allows users to define custom labels for metrics.
// The value can use placeholders like {http.request.scheme}, {http.request.proto}, {http.request.remote}, etc.
// These labels will be added to all HTTP metrics.
Labels map[string]string `json:"labels,omitempty"`

init sync.Once
httpMetrics *httpMetrics `json:"-"`
}
Expand All @@ -44,6 +49,12 @@ func initHTTPMetrics(ctx caddy.Context, metrics *Metrics) {
if metrics.PerHost {
basicLabels = append(basicLabels, "host")
}
if metrics.Labels != nil {
for key := range metrics.Labels {
basicLabels = append(basicLabels, key)
}
}

metrics.httpMetrics.requestInFlight = promauto.With(registry).NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: sub,
Expand Down Expand Up @@ -71,6 +82,12 @@ func initHTTPMetrics(ctx caddy.Context, metrics *Metrics) {
if metrics.PerHost {
httpLabels = append(httpLabels, "host")
}
if metrics.Labels != nil {
for key := range metrics.Labels {
httpLabels = append(httpLabels, key)
}
}

metrics.httpMetrics.requestDuration = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
Namespace: ns,
Subsystem: sub,
Expand Down Expand Up @@ -111,6 +128,36 @@ func serverNameFromContext(ctx context.Context) string {
return srv.name
}

// processCustomLabels processes custom labels by replacing placeholders with actual values.
func (h *metricsInstrumentedHandler) processCustomLabels(r *http.Request) prometheus.Labels {
labels := make(prometheus.Labels)

if h.metrics.Labels == nil {
return labels
}

repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
if repl == nil {
repl = caddy.NewReplacer()
}

for key, value := range h.metrics.Labels {
if strings.Contains(value, "{") && strings.Contains(value, "}") {
replaced := repl.ReplaceAll(value, "")

if replaced == "" || replaced == value {
replaced = "unknown"
}

labels[key] = replaced
} else {
labels[key] = value
}
}

return labels
}

type metricsInstrumentedHandler struct {
handler string
mh MiddlewareHandler
Expand Down Expand Up @@ -138,6 +185,12 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
statusLabels["host"] = strings.ToLower(r.Host)
}

customLabels := h.processCustomLabels(r)
for key, value := range customLabels {
labels[key] = value
statusLabels[key] = value
}

Comment on lines +188 to +193
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I'm afraid of, happened. I've just tested and all the replaced labels have the value "unknown". Applying replacements here is too early because many of the values in the replacer aren't filled in yet. Can this be moved lower or refactored so it picks up the labeling? I imagine it needs something similar to observeRequest.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you tell me the labels key that is being recorded as unknown at that point? i thought the only value net yet determined at that point was status, and that status is determined in the code below, so i thought there was no problem.
but i'm not familiar with this code base, so if there's something i haven't discovered yet, plz let me know for reproducing test.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use any of the placeholders here or here or set via vars handler, and you'll find they're all reported as unknown.

inFlight := h.metrics.httpMetrics.requestInFlight.With(labels)
inFlight.Inc()
defer inFlight.Dec()
Expand Down
76 changes: 76 additions & 0 deletions modules/caddyhttp/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,82 @@ func TestMetricsInstrumentedHandlerPerHost(t *testing.T) {
}
}

func TestMetricsInstrumentedHandlerCustomLabels(t *testing.T) {
ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
metrics := &Metrics{
Labels: map[string]string{
"proto": "{http.request.proto}",
"client_ip": "IP: {http.request.remote}",
"host": "Host is {http.request.host}",
"version": "v1.0.0",
},
init: sync.Once{},
httpMetrics: &httpMetrics{},
}

h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
w.Write([]byte("hello world!"))
return nil
})

mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
return h.ServeHTTP(w, r)
})

ih := newMetricsInstrumentedHandler(ctx, "custom_labels", mh, metrics)

r := httptest.NewRequest("GET", "/", nil)
r.Host = "example.com"
r.RemoteAddr = "192.168.1.1:12345"

repl := caddy.NewReplacer()
reqCtx := context.WithValue(r.Context(), caddy.ReplacerCtxKey, repl)
r = r.WithContext(reqCtx)

w := httptest.NewRecorder()

addHTTPVarsToReplacer(repl, r, w)

if err := ih.ServeHTTP(w, r, h); err != nil {
t.Errorf("Received unexpected error: %v", err)
}

expected := `
# HELP caddy_http_request_size_bytes Total size of the request. Includes body
# TYPE caddy_http_request_size_bytes histogram
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="256"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="1024"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="4096"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="16384"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="65536"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="262144"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="1.048576e+06"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="4.194304e+06"} 1
caddy_http_request_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="+Inf"} 1
caddy_http_request_size_bytes_sum{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0"} 23
caddy_http_request_size_bytes_count{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0"} 1
# HELP caddy_http_response_size_bytes Size of the returned response.
# TYPE caddy_http_response_size_bytes histogram
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="256"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="1024"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="4096"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="16384"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="65536"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="262144"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="1.048576e+06"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="4.194304e+06"} 1
caddy_http_response_size_bytes_bucket{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0",le="+Inf"} 1
caddy_http_response_size_bytes_sum{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0"} 12
caddy_http_response_size_bytes_count{client_ip="IP: 192.168.1.1:12345",code="200",handler="custom_labels",host="Host is example.com",method="GET",proto="HTTP/1.1",server="UNKNOWN",version="v1.0.0"} 1
`
if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
"caddy_http_request_size_bytes",
"caddy_http_response_size_bytes",
); err != nil {
t.Errorf("received unexpected error: %s", err)
}
}

type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error

func (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error {
Expand Down
Loading