feat: Add support for tracing

Signed-off-by: Bilal Hussain <[email protected]>

CHANGELOG.md

@@ -23,6 +23,7 @@ This changelog keeps track of work items that have been completed and are ready
 
 ### New
 
+- **General**: Add configurable tracing support to the interceptor proxy ([#1021](https://github.com/kedacore/http-add-on/pull/1021))
 - **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO))
 
 ### Improvements

Makefile

@@ -199,4 +199,4 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 undeploy:
-	$(KUSTOMIZE) build config/default | kubectl delete -f -
+	$(KUSTOMIZE) build config/default | kubectl delete -f -

config/interceptor/e2e-test/otel/deployment.yaml

@@ -21,3 +21,11 @@ spec:
           value: "true"
         - name: KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL
           value: "1"
+        - name: KEDA_HTTP_TRACING_ENABLED
+          value: "true"
+        - name: KEDA_HTTP_TRACE_EXPORTER
+          value: otlphttp
+        - name: KEDA_HTTP_TRACE_ENDPOINT
+          value: "opentelemetry-collector.open-telemetry-system:4318"
+        - name: KEDA_HTTP_TRACE_INSECURE
+          value: "true"

docs/operate.md

@@ -25,3 +25,28 @@ If you need to provide any headers such as authentication details in order to ut
 The interceptor proxy has the ability to run both a HTTP and HTTPS server simultaneously to allow you to scale workloads that use either protocol. By default, the interceptor proxy will only serve over HTTP, but this behavior can be changed by configuring the appropriate environment variables on the deployment.
 
 The TLS server can be enabled by setting the environment variable `KEDA_HTTP_PROXY_TLS_ENABLED` to `true` on the interceptor deployment (`false` by default). The TLS server will start on port `8443` by default, but this can be configured by setting `KEDA_HTTP_PROXY_TLS_PORT` to your desired port number. The TLS server will require valid TLS certificates to start, the path to the certificates can be configured via the `KEDA_HTTP_PROXY_TLS_CERT_PATH` and `KEDA_HTTP_PROXY_TLS_KEY_PATH` environment variables (`/certs/tls.crt` and `/certs/tls.key` by default).
+
+# Configuring tracing for the KEDA HTTP Add-on interceptor proxy
+
+### Supported Exporters:
+* **console** - The console exporter is useful for development and debugging tasks, and is the simplest to set up.
+* **otlphttp** - To send trace data to an OTLP endpoint (like the collector or Jaeger >= v1.35.0) you’ll want to configure an OTLP exporter that sends to your endpoint.
+
+### Configuring tracing with console exporter
+
+To enable tracing with the console exporter, the `KEDA_HTTP_TRACING_ENABLED` environment variable should be set to `true` on the interceptor deployment. (`false` by default).
+Secondly set `KEDA_HTTP_TRACE_EXPORTER` to `console` (`console` by default).
+Finally set `KEDA_HTTP_TRACE_ENDPOINT` to `"localhost:4318"` (`"localhost:4318"` by default).
+
+
+### Configuring tracing with OTLP exporter
+When configured, the interceptor proxy can export metrics to a OTEL HTTP collector.
+
+To enable tracing with otlp exporter, the `KEDA_HTTP_TRACING_ENABLED` environment variable should be set to `true` on the interceptor deployment. (`false` by default).
+Secondly set `KEDA_HTTP_TRACE_EXPORTER` to `otlphttp` (`console` by default).
+Finally set `KEDA_HTTP_TRACE_ENDPOINT` to the collector to send the traces to (e.g. opentelemetry-collector.open-telemetry-system:4318) (`"localhost:4318"` by default).
+
+Optional variables
+`OTEL_EXPORTER_OTLP_HEADERS` - To pass any extra headers to the spans to utilise your OTEL collector e.g. authentication details (`"key1=value1,key2=value2"`)
+`KEDA_HTTP_TRACE_INSECURE` - To send traces to the tracing via HTTP rather than HTTPS (`false` by default)
+`KEDA_HTTP_BATCH_TIMEOUT` - The batcher timeout in seconds to send batch of data points (`5` by default)

go.mod

@@ -11,9 +11,13 @@ require (
 	github.com/onsi/ginkgo/v2 v2.17.2
 	github.com/onsi/gomega v1.33.0
 	github.com/stretchr/testify v1.9.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
 	go.opentelemetry.io/otel v1.26.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0
 	go.opentelemetry.io/otel/sdk v1.26.0
+	go.opentelemetry.io/otel/trace v1.26.0
 	go.uber.org/mock v0.4.0
 	golang.org/x/sync v0.7.0
 	google.golang.org/grpc v1.63.2
@@ -43,12 +47,12 @@ replace (
 require (
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect
@@ -99,6 +103,7 @@ require (
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/prometheus v0.45.1
 	go.opentelemetry.io/otel/metric v1.26.0
 	go.opentelemetry.io/otel/sdk/metric v1.26.0

go.sum

@@ -21,6 +21,8 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/expr-lang/expr v1.16.5 h1:m2hvtguFeVaVNTHj8L7BoAyt7O0PAIBaSVbjdHgRXMs=
 github.com/expr-lang/expr v1.16.5/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
@@ -147,12 +149,20 @@ github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
 github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
 go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
 go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0 h1:HGZWGmCVRCVyAs2GQaiHQPbDHo+ObFWeUEOd+zDnp64=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0/go.mod h1:SaH+v38LSCHddyk7RGlU9uZyQoRrKao6IBnJw6Kbn+c=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM=
 go.opentelemetry.io/otel/exporters/prometheus v0.45.1 h1:R/bW3afad6q6VGU+MFYpnEdo0stEARMCdhWu6+JI6aI=
 go.opentelemetry.io/otel/exporters/prometheus v0.45.1/go.mod h1:wnHAfKRav5Dfp4iZhyWZ7SzQfT+rDZpEpYG7To+qJ1k=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 h1:s0PHtIkN+3xrbDOpt2M8OTG92cWqUESvzh2MxiR5xY8=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0/go.mod h1:hZlFbDbRt++MMPCCfSJfmhkGIWnX1h3XjkfxZUjLrIA=
 go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
 go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
 go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8=

interceptor/config/tracing.go

@@ -0,0 +1,29 @@
+package config
+
+import (
+	"github.com/kelseyhightower/envconfig"
+)
+
+// Tracing is the configuration for configuring tracing through the interceptor.
+type Tracing struct {
+	// States whether tracing should be enabled, False by default
+	Enabled bool `envconfig:"KEDA_HTTP_TRACING_ENABLED" default:"false"`
+	// Sets what tracing export to use, must be one of: console,otlphttp
+	Exporter string `envconfig:"KEDA_HTTP_TRACE_EXPORTER" default:"console"`
+	// Sets the endpoint which traces should be forwarded to
+	Endpoint string `envconfig:"KEDA_HTTP_TRACE_ENDPOINT" default:"localhost:4318"`
+	// OTLP headers
+	Headers string `envconfig:"OTEL_EXPORTER_OTLP_HEADERS" default:""`
+	// Sends traces to the tracing via HTTP rather than HTTPS. Defaults to False
+	Insecure bool `envconfig:"KEDA_HTTP_TRACE_INSECURE" default:"false"`
+	// The timeout for the batch processor is described as "Time duration after which a batch will be sent regardless of size."
+	Timeout int `envconfig:"KEDA_HTTP_BATCH_TIMEOUT" default:"5"`
+}
+
+// Parse parses standard configs using envconfig and returns a pointer to the
+// newly created config. Returns nil and a non-nil error if parsing failed
+func MustParseTracing() *Tracing {
+	ret := new(Tracing)
+	envconfig.MustProcess("", ret)
+	return ret
+}

interceptor/handler/upstream.go

@@ -5,6 +5,13 @@ import (
 	"net/http"
 	"net/http/httputil"
 
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/kedacore/http-add-on/interceptor/config"
+	"github.com/kedacore/http-add-on/interceptor/tracing"
 	"github.com/kedacore/http-add-on/pkg/util"
 )
 
@@ -14,11 +21,13 @@ var (
 
 type Upstream struct {
 	roundTripper http.RoundTripper
+	tracingCfg   *config.Tracing
 }
 
-func NewUpstream(roundTripper http.RoundTripper) *Upstream {
+func NewUpstream(roundTripper http.RoundTripper, tracingCfg *config.Tracing) *Upstream {
 	return &Upstream{
 		roundTripper: roundTripper,
+		tracingCfg:   tracingCfg,
 	}
 }
 
@@ -28,6 +37,20 @@ func (uh *Upstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	r = util.RequestWithLoggerWithName(r, "UpstreamHandler")
 	ctx := r.Context()
 
+	if uh.tracingCfg.Enabled {
+		p := tracing.NewPropagator()
+		ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header))
+		p.Inject(ctx, propagation.HeaderCarrier(w.Header()))
+
+		span := trace.SpanFromContext(ctx)
+		defer span.End()
+
+		serviceValAttr := attribute.String("service", "keda-http-interceptor-proxy-upstream")
+		coldStartValAttr := attribute.String("cold-start", w.Header().Get("X-KEDA-HTTP-Cold-Start"))
+
+		span.SetAttributes(serviceValAttr, coldStartValAttr)
+	}
+
 	stream := util.StreamFromContext(ctx)
 	if stream == nil {
 		sh := NewStatic(http.StatusInternalServerError, errNilStream)

interceptor/handler/upstream_test.go

@@ -43,7 +43,7 @@ func TestForwarderSuccess(t *testing.T) {
 	timeouts := defaultTimeouts()
 	dialCtxFunc := retryDialContextFunc(timeouts, timeouts.DefaultBackoff())
 	rt := newRoundTripper(dialCtxFunc, timeouts.ResponseHeader)
-	uh := NewUpstream(rt)
+	uh := NewUpstream(rt, &config.Tracing{})
 	uh.ServeHTTP(res, req)
 
 	r.True(
@@ -88,7 +88,7 @@ func TestForwarderHeaderTimeout(t *testing.T) {
 	r.NoError(err)
 	req = util.RequestWithStream(req, originURL)
 	rt := newRoundTripper(dialCtxFunc, timeouts.ResponseHeader)
-	uh := NewUpstream(rt)
+	uh := NewUpstream(rt, &config.Tracing{})
 	uh.ServeHTTP(res, req)
 
 	forwardedRequests := hdl.IncomingRequests()
@@ -138,7 +138,7 @@ func TestForwarderWaitsForSlowOrigin(t *testing.T) {
 	r.NoError(err)
 	req = util.RequestWithStream(req, originURL)
 	rt := newRoundTripper(dialCtxFunc, timeouts.ResponseHeader)
-	uh := NewUpstream(rt)
+	uh := NewUpstream(rt, &config.Tracing{})
 	uh.ServeHTTP(res, req)
 	// wait for the goroutine above to finish, with a little cusion
 	ensureSignalBeforeTimeout(originWaitCh, originDelay*2)
@@ -161,7 +161,7 @@ func TestForwarderConnectionRetryAndTimeout(t *testing.T) {
 	r.NoError(err)
 	req = util.RequestWithStream(req, noSuchURL)
 	rt := newRoundTripper(dialCtxFunc, timeouts.ResponseHeader)
-	uh := NewUpstream(rt)
+	uh := NewUpstream(rt, &config.Tracing{})
 
 	start := time.Now()
 	uh.ServeHTTP(res, req)
@@ -217,7 +217,7 @@ func TestForwardRequestRedirectAndHeaders(t *testing.T) {
 	r.NoError(err)
 	req = util.RequestWithStream(req, srvURL)
 	rt := newRoundTripper(dialCtxFunc, timeouts.ResponseHeader)
-	uh := NewUpstream(rt)
+	uh := NewUpstream(rt, &config.Tracing{})
 	uh.ServeHTTP(res, req)
 	r.Equal(301, res.Code)
 	r.Equal("abc123.com", res.Header().Get("Location"))

interceptor/main.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -22,6 +23,7 @@ import (
 	"github.com/kedacore/http-add-on/interceptor/handler"
 	"github.com/kedacore/http-add-on/interceptor/metrics"
 	"github.com/kedacore/http-add-on/interceptor/middleware"
+	"github.com/kedacore/http-add-on/interceptor/tracing"
 	clientset "github.com/kedacore/http-add-on/operator/generated/clientset/versioned"
 	informers "github.com/kedacore/http-add-on/operator/generated/informers/externalversions"
 	"github.com/kedacore/http-add-on/pkg/build"
@@ -44,6 +46,7 @@ func main() {
 	timeoutCfg := config.MustParseTimeouts()
 	servingCfg := config.MustParseServing()
 	metricsCfg := config.MustParseMetrics()
+	tracingCfg := config.MustParseTracing()
 
 	opts := zap.Options{
 		Development: true,
@@ -115,6 +118,18 @@ func main() {
 
 	eg, ctx := errgroup.WithContext(ctx)
 
+	if tracingCfg.Enabled {
+		shutdown, err := tracing.SetupOTelSDK(ctx, tracingCfg)
+
+		if err != nil {
+			setupLog.Error(err, "Error setting up tracer")
+		}
+
+		defer func() {
+			err = errors.Join(err, shutdown(context.Background()))
+		}()
+	}
+
 	// start the endpoints cache updater
 	eg.Go(func() error {
 		setupLog.Info("starting the endpoints cache")
@@ -173,7 +188,7 @@ func main() {
 
 			setupLog.Info("starting the proxy server with TLS enabled", "port", proxyTLSPort)
 
-			if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyTLSPort, proxyTLSEnabled, proxyTLSConfig); !util.IsIgnoredErr(err) {
+			if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyTLSPort, proxyTLSEnabled, proxyTLSConfig, tracingCfg); !util.IsIgnoredErr(err) {
 				setupLog.Error(err, "tls proxy server failed")
 				return err
 			}
@@ -185,7 +200,7 @@ func main() {
 	eg.Go(func() error {
 		setupLog.Info("starting the proxy server with TLS disabled", "port", proxyPort)
 
-		if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyPort, false, nil); !util.IsIgnoredErr(err) {
+		if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyPort, false, nil, tracingCfg); !util.IsIgnoredErr(err) {
 			setupLog.Error(err, "proxy server failed")
 			return err
 		}
@@ -242,6 +257,7 @@ func runProxyServer(
 	port int,
 	tlsEnabled bool,
 	tlsConfig map[string]string,
+	tracingConfig *config.Tracing,
 ) error {
 	dialer := kedanet.NewNetDialer(timeouts.Connect, timeouts.KeepAlive)
 	dialContextFunc := kedanet.DialContextWithRetry(dialer, timeouts.DefaultBackoff())
@@ -278,6 +294,7 @@ func runProxyServer(
 		waitFunc,
 		newForwardingConfigFromTimeouts(timeouts),
 		&tlsCfg,
+		tracingConfig,
 	)
 	upstreamHandler = middleware.NewCountingMiddleware(
 		q,
@@ -291,6 +308,11 @@ func runProxyServer(
 		upstreamHandler,
 		tlsEnabled,
 	)
+
+	if tracingConfig.Enabled {
+		rootHandler = otelhttp.NewHandler(rootHandler, "keda-http-interceptor")
+	}
+
 	rootHandler = middleware.NewLogging(
 		logger,
 		rootHandler,

interceptor/main_test.go

@@ -16,6 +16,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/kedacore/http-add-on/interceptor/config"
+	"github.com/kedacore/http-add-on/interceptor/tracing"
 	"github.com/kedacore/http-add-on/pkg/k8s"
 	kedanet "github.com/kedacore/http-add-on/pkg/net"
 	"github.com/kedacore/http-add-on/pkg/queue"
@@ -68,6 +69,15 @@ func TestRunProxyServerCountMiddleware(t *testing.T) {
 		<-waiterCh
 		return false, nil
 	}
+
+	tracingCfg := config.Tracing{Enabled: true, Exporter: "otlphttp", Endpoint: "localhost:4318"}
+
+	_, err = tracing.SetupOTelSDK(ctx, &tracingCfg)
+
+	if err != nil {
+		fmt.Println(err, "Error setting up tracer")
+	}
+
 	g.Go(func() error {
 		return runProxyServer(
 			ctx,
@@ -79,6 +89,7 @@ func TestRunProxyServerCountMiddleware(t *testing.T) {
 			port,
 			false,
 			map[string]string{},
+			&tracingCfg,
 		)
 	})
 	// wait for server to start
@@ -107,6 +118,10 @@ func TestRunProxyServerCountMiddleware(t *testing.T) {
 				resp.StatusCode,
 			)
 		}
+		if _, ok := resp.Header["Traceparent"]; !ok {
+			return fmt.Errorf("expected Traceparent header to exist, but the header wasn't found")
+		}
+
 		if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != "false" {
 			return fmt.Errorf("expected X-KEDA-HTTP-Cold-Start false, but got %s", resp.Header.Get("X-KEDA-HTTP-Cold-Start"))
 		}
@@ -199,6 +214,7 @@ func TestRunProxyServerWithTLSCountMiddleware(t *testing.T) {
 		<-waiterCh
 		return false, nil
 	}
+	tracingCfg := config.Tracing{Enabled: true, Exporter: "otlphttp", Endpoint: "localhost:4318"}
 
 	g.Go(func() error {
 		return runProxyServer(
@@ -211,6 +227,7 @@ func TestRunProxyServerWithTLSCountMiddleware(t *testing.T) {
 			port,
 			true,
 			map[string]string{"certificatePath": "../certs/tls.crt", "keyPath": "../certs/tls.key"},
+			&tracingCfg,
 		)
 	})