diff --git a/.github/workflows/apps/go.mod b/.github/workflows/apps/go.mod new file mode 100644 index 0000000000..19c2d9dfde --- /dev/null +++ b/.github/workflows/apps/go.mod @@ -0,0 +1,16 @@ +module github.com/DataDog/dd-trace-go/.github/workflows/apps + +go 1.23.3 + +require ( + github.com/Masterminds/semver/v3 v3.3.1 + github.com/stretchr/testify v1.10.0 + golang.org/x/mod v0.22.0 + gopkg.in/DataDog/dd-trace-go.v1 v1.70.1 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/.github/workflows/apps/go.sum b/.github/workflows/apps/go.sum new file mode 100644 index 0000000000..4958ac6c0a --- /dev/null +++ b/.github/workflows/apps/go.sum @@ -0,0 +1,16 @@ +github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= +github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +gopkg.in/DataDog/dd-trace-go.v1 v1.70.1 h1:ZIRxAKlr3xr6xbMUDs3IDa6xq+ISv9zxyjaDCfwDjMY= +gopkg.in/DataDog/dd-trace-go.v1 v1.70.1/go.mod h1:PMOSkeY4VfXiuPvGodeNLCZCFYU2VfOvjVI6cX5bGrc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.github/workflows/apps/latest_major_version.go b/.github/workflows/apps/latest_major_version.go new file mode 100644 index 0000000000..f471afe580 --- /dev/null +++ b/.github/workflows/apps/latest_major_version.go @@ -0,0 +1,153 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package main + +import ( + "encoding/json" + "fmt" + "sort" + "github.com/Masterminds/semver/v3" + "golang.org/x/mod/modfile" + "net/http" + "os" + "regexp" + "strings" +) + +type Tag struct { + Name string +} + +func getLatestMajorVersion(repo string) (string, error) { + // Get latest major version available for repo from github. + const apiURL = "https://api.github.com/repos/%s/tags" + url := fmt.Sprintf(apiURL, repo) + + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch tags: %s", resp.Status) + } + + var tags []struct { + Name string `json:"name"` + } + + if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil { + return "", err + } + latestByMajor := make(map[int]*semver.Version) + + for _, tag := range tags { + v, err := semver.NewVersion(tag.Name) + if err != nil { + continue // Skip invalid versions + } + + if v.Prerelease() != "" { + continue // Ignore pre-release versions + } + + major := int(v.Major()) + if current, exists := latestByMajor[major]; !exists || v.GreaterThan(current) { + latestByMajor[major] = v + } + } + + var latestMajor *semver.Version + for _, v := range latestByMajor { + if latestMajor == nil || v.Major() > latestMajor.Major() { + latestMajor = v + } + } + + if latestMajor != nil { + return fmt.Sprintf("v%d", latestMajor.Major()), nil + } + + return "", fmt.Errorf("no valid versions found") + +} + +func main() { + + data, err := os.ReadFile("integration_go.mod") + if err != nil { + fmt.Println("Error reading integration_go.mod:", err) + return + } + + modFile, err := modfile.Parse("integration_go.mod", data, nil) + if err != nil { + fmt.Println("Error parsing integration_go.mod:", err) + return + } + + latestMajor := make(map[string]*semver.Version) + + // Match on versions with /v{major} + versionRegex := regexp.MustCompile(`^(?P.+?)/v(\d+)$`) + + // Iterate over the required modules and update latest major version if necessary + for _, req := range modFile.Require { + module := req.Mod.Path + + if match := versionRegex.FindStringSubmatch(module); match != nil { + url := match[1] // base module URL (e.g., github.com/foo) + majorVersionStr := "v" + match[2] // Create semantic version string (e.g., "v2") + + moduleName := strings.TrimPrefix(strings.TrimSpace(url), "github.com/") + + // Parse the semantic version + majorVersion, err := semver.NewVersion(majorVersionStr) + if err != nil { + fmt.Printf("Skip invalid version for module %s: %v\n", module, err) + continue + } + + if existing, ok := latestMajor[moduleName]; !ok || majorVersion.GreaterThan(existing) { + latestMajor[moduleName] = majorVersion + } + } + } + + // Output latest major version that we support. + // Check if a new major version in Github is available that we don't support. + // If so, output that a new latest is available. + + // Sort the output + modules := make([]string, 0, len(latestMajor)) + for module := range latestMajor { + modules = append(modules, module) + } + sort.Strings(modules) + + for _, module := range modules { + major := latestMajor[module] + + latestVersion, err := getLatestMajorVersion(module) // latest version available + if err != nil { + fmt.Printf("Error fetching latest version for module '%s': %v\n", module, err) + continue + } + + latestVersionParsed, err := semver.NewVersion(latestVersion) + if err != nil { + fmt.Printf("Error parsing latest version '%s' for module '%s': %v\n", latestVersion, module, err) + continue + } + + fmt.Printf("Latest DD major version of %s: %d\n", module, major.Major()) + if major.LessThan(latestVersionParsed) { + fmt.Printf("New latest major version of %s available: %d\n", module, latestVersionParsed.Major()) + } + + } +} diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 80c1fce6cf..d238b55ba3 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -19,6 +19,9 @@ on: merge_group: push: branches: release-v* + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' env: DD_APPSEC_WAF_TIMEOUT: 1m diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8d85bac9df..53fb2d4cc2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,6 +9,9 @@ on: type: string push: branches: [ main, master ] + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' pull_request: # The branches below must be a subset of the branches above branches: [ main ] diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index eaf5ed78d9..e7549a71fc 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -9,7 +9,10 @@ on: push: branches: - main - - release-v* + - release-v* + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' schedule: - cron: '00 00 * * *' workflow_dispatch: diff --git a/.github/workflows/main-branch-tests.yml b/.github/workflows/main-branch-tests.yml index 7d738ccd9a..136805cebd 100644 --- a/.github/workflows/main-branch-tests.yml +++ b/.github/workflows/main-branch-tests.yml @@ -11,8 +11,9 @@ on: branches: - main - release-v* - tags: - - "**" + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' concurrency: group: ${{ github.ref }} diff --git a/.github/workflows/orchestrion.yml b/.github/workflows/orchestrion.yml index 0603c8b139..ddd0e7435b 100644 --- a/.github/workflows/orchestrion.yml +++ b/.github/workflows/orchestrion.yml @@ -6,6 +6,9 @@ on: push: branches: - release-v* + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' permissions: read-all diff --git a/.github/workflows/outdated-integrations.yml b/.github/workflows/outdated-integrations.yml new file mode 100644 index 0000000000..0e559c8039 --- /dev/null +++ b/.github/workflows/outdated-integrations.yml @@ -0,0 +1,41 @@ +name: Outdated Integrations +on: + schedule: + - cron: "0 0 * * 0" # Runs every Sunday at midnight UTC + workflow_dispatch: + +concurrency: + # Automatically cancel previous runs if a new one is triggered to conserve resources. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Find new major versions for the contrib package dependencies + runs-on: ubuntu-latest + permissions: + actions: read + contents: write + pull-requests: write + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - run: go get github.com/Masterminds/semver/v3 + + - run: go run .github/workflows/apps/latest_major_version.go > latests.txt + + - run: git diff + + - name: Create Pull Request + id: pr + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: "upgrade-latest-major-version" + commit-message: "Update latests file" + base: main + title: "chore: update latest majors" + labels: changelog/no-changelog + body: "Auto-generated PR from Outdated Integrations workflow to update latests major versions" diff --git a/.github/workflows/parametric-tests.yml b/.github/workflows/parametric-tests.yml index a25c01a2c3..4027451800 100644 --- a/.github/workflows/parametric-tests.yml +++ b/.github/workflows/parametric-tests.yml @@ -11,8 +11,9 @@ on: branches: - main - release-v* - tags: - - "**" + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' pull_request: branches: - "**" diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 9513b2e328..8318181820 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -8,6 +8,9 @@ on: push: branches: - 'mq-working-branch-**' + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' concurrency: group: ${{ github.ref }} diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index cee00b0d34..9a05bdc8ba 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -15,8 +15,9 @@ on: branches: - main - release-v* - tags: - - '**' + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' schedule: # nightly - cron: "0 0 * * *" workflow_dispatch: {} # manually diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 90933ec1ad..f744ce1303 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -11,8 +11,9 @@ on: branches: - main - release-v* - tags: - - "**" + tags-ignore: + - 'contrib/**' + - 'instrumentation/**' pull_request: branches: - "**" diff --git a/contrib/IBM/sarama.v1/sarama.go b/contrib/IBM/sarama.v1/sarama.go index ef879b0228..f684cf48e1 100644 --- a/contrib/IBM/sarama.v1/sarama.go +++ b/contrib/IBM/sarama.v1/sarama.go @@ -74,6 +74,10 @@ func WrapPartitionConsumer(pc sarama.PartitionConsumer, opts ...Option) sarama.P // kafka supports headers, so try to extract a span context carrier := NewConsumerMessageCarrier(msg) if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } next := tracer.StartSpan(cfg.consumerSpanName, opts...) @@ -298,6 +302,10 @@ func startProducerSpan(cfg *config, version sarama.KafkaVersion, msg *sarama.Pro } // if there's a span context in the headers, use that as the parent if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } span := tracer.StartSpan(cfg.producerSpanName, opts...) diff --git a/contrib/Shopify/sarama/sarama.go b/contrib/Shopify/sarama/sarama.go index 6c3a8c6d71..d34d873597 100644 --- a/contrib/Shopify/sarama/sarama.go +++ b/contrib/Shopify/sarama/sarama.go @@ -77,6 +77,10 @@ func WrapPartitionConsumer(pc sarama.PartitionConsumer, opts ...Option) sarama.P // kafka supports headers, so try to extract a span context carrier := NewConsumerMessageCarrier(msg) if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } next := tracer.StartSpan(cfg.consumerSpanName, opts...) @@ -301,6 +305,10 @@ func startProducerSpan(cfg *config, version sarama.KafkaVersion, msg *sarama.Pro } // if there's a span context in the headers, use that as the parent if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } span := tracer.StartSpan(cfg.producerSpanName, opts...) diff --git a/contrib/cloud.google.com/go/pubsub.v1/internal/tracing/tracing.go b/contrib/cloud.google.com/go/pubsub.v1/internal/tracing/tracing.go index 7948db1a59..ab0b82f07f 100644 --- a/contrib/cloud.google.com/go/pubsub.v1/internal/tracing/tracing.go +++ b/contrib/cloud.google.com/go/pubsub.v1/internal/tracing/tracing.go @@ -118,6 +118,10 @@ func TraceReceiveFunc(s Subscription, opts ...Option) func(ctx context.Context, if cfg.measured { opts = append(opts, tracer.Measured()) } + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := parentSpanCtx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } span, ctx := tracer.StartSpanFromContext(ctx, cfg.receiveSpanName, opts...) if msg.DeliveryAttempt != nil { span.SetTag("delivery_attempt", *msg.DeliveryAttempt) diff --git a/contrib/confluentinc/confluent-kafka-go/internal/tracing/consumer.go b/contrib/confluentinc/confluent-kafka-go/internal/tracing/consumer.go index 90678c4ed2..aca6125aa7 100644 --- a/contrib/confluentinc/confluent-kafka-go/internal/tracing/consumer.go +++ b/contrib/confluentinc/confluent-kafka-go/internal/tracing/consumer.go @@ -77,6 +77,10 @@ func (tr *KafkaTracer) StartConsumeSpan(msg Message) ddtrace.Span { // kafka supports headers, so try to extract a span context carrier := MessageCarrier{msg: msg} if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } span, _ := tracer.StartSpanFromContext(tr.ctx, tr.consumerSpanName, opts...) diff --git a/contrib/confluentinc/confluent-kafka-go/internal/tracing/producer.go b/contrib/confluentinc/confluent-kafka-go/internal/tracing/producer.go index 25b043017f..e7e78db042 100644 --- a/contrib/confluentinc/confluent-kafka-go/internal/tracing/producer.go +++ b/contrib/confluentinc/confluent-kafka-go/internal/tracing/producer.go @@ -67,6 +67,10 @@ func (tr *KafkaTracer) StartProduceSpan(msg Message) ddtrace.Span { // if there's a span context in the headers, use that as the parent carrier := NewMessageCarrier(msg) if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } span, _ := tracer.StartSpanFromContext(tr.ctx, tr.producerSpanName, opts...) diff --git a/contrib/database/sql/option_test.go b/contrib/database/sql/option_test.go index e04c5ef452..469bc48736 100644 --- a/contrib/database/sql/option_test.go +++ b/contrib/database/sql/option_test.go @@ -69,7 +69,7 @@ func TestCheckStatsdRequired(t *testing.T) { cfg := new(config) cfg.dbStats = true cfg.checkStatsdRequired() - _, ok := cfg.statsdClient.(*statsd.Client) + _, ok := cfg.statsdClient.(*statsd.ClientDirect) assert.True(t, ok) }) t.Run("invalid address", func(t *testing.T) { diff --git a/contrib/envoyproxy/go-control-plane/envoy.go b/contrib/envoyproxy/go-control-plane/envoy.go new file mode 100644 index 0000000000..52279e0138 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/envoy.go @@ -0,0 +1,350 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane + +import ( + "context" + "errors" + "io" + "math" + "net/http" + "strings" + + "gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/emitter/waf/actions" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + "gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoyextproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoytypes "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +const componentName = "envoyproxy/go-control-plane" + +func init() { + telemetry.LoadIntegration(componentName) + tracer.MarkIntegrationImported("github.com/envoyproxy/go-control-plane") +} + +// appsecEnvoyExternalProcessorServer is a server that implements the Envoy ExternalProcessorServer interface. +type appsecEnvoyExternalProcessorServer struct { + envoyextproc.ExternalProcessorServer +} + +// AppsecEnvoyExternalProcessorServer creates and returns a new instance of appsecEnvoyExternalProcessorServer. +func AppsecEnvoyExternalProcessorServer(userImplementation envoyextproc.ExternalProcessorServer) envoyextproc.ExternalProcessorServer { + return &appsecEnvoyExternalProcessorServer{userImplementation} +} + +type currentRequest struct { + span tracer.Span + afterHandle func() + ctx context.Context + fakeResponseWriter *fakeResponseWriter + wrappedResponseWriter http.ResponseWriter +} + +// Process handles the bidirectional stream that Envoy uses to give the server control +// over what the filter does. It processes incoming requests and sends appropriate responses +// based on the type of request received. +// +// The method receive incoming requests, processes them, and sends responses back to the client. +// It handles different types of requests such as request headers, response headers, request body, +// response body, request trailers, and response trailers. +// +// If the request is blocked, it sends an immediate response and ends the stream. If an error occurs +// during processing, it logs the error and returns an appropriate gRPC status error. +func (s *appsecEnvoyExternalProcessorServer) Process(processServer envoyextproc.ExternalProcessor_ProcessServer) error { + var ( + ctx = processServer.Context() + blocked bool + currentRequest *currentRequest + processingRequest envoyextproc.ProcessingRequest + processingResponse *envoyextproc.ProcessingResponse + ) + + // Close the span when the request is done processing + defer func() { + if currentRequest == nil { + return + } + + log.Warn("external_processing: stream stopped during a request, making sure the current span is closed\n") + currentRequest.span.Finish() + currentRequest = nil + }() + + for { + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + return nil + } + + return ctx.Err() + default: + // no op + } + + err := processServer.RecvMsg(&processingRequest) + if err != nil { + // Note: Envoy is inconsistent with the "end_of_stream" value of its headers responses, + // so we can't fully rely on it to determine when it will close (cancel) the stream. + if s, ok := status.FromError(err); (ok && s.Code() == codes.Canceled) || err == io.EOF { + return nil + } + + log.Warn("external_processing: error receiving request/response: %v\n", err) + return status.Errorf(codes.Unknown, "Error receiving request/response: %v", err) + } + + processingResponse, err = envoyExternalProcessingRequestTypeAssert(&processingRequest) + if err != nil { + log.Error("external_processing: error asserting request type: %v\n", err) + return status.Errorf(codes.Unknown, "Error asserting request type: %v", err) + } + + switch v := processingRequest.Request.(type) { + case *envoyextproc.ProcessingRequest_RequestHeaders: + processingResponse, currentRequest, blocked, err = processRequestHeaders(ctx, v) + case *envoyextproc.ProcessingRequest_ResponseHeaders: + processingResponse, err = processResponseHeaders(v, currentRequest) + currentRequest = nil // Request is done, reset the current request + } + + if err != nil { + log.Error("external_processing: error processing request: %v\n", err) + return err + } + + // End of stream reached, no more data to process + if processingResponse == nil { + log.Debug("external_processing: end of stream reached") + return nil + } + + if err := processServer.SendMsg(processingResponse); err != nil { + log.Warn("external_processing: error sending response (probably because of an Envoy timeout): %v", err) + return status.Errorf(codes.Unknown, "Error sending response (probably because of an Envoy timeout): %v", err) + } + + if !blocked { + continue + } + + log.Debug("external_processing: request blocked, end the stream") + currentRequest = nil + return nil + } +} + +func envoyExternalProcessingRequestTypeAssert(req *envoyextproc.ProcessingRequest) (*envoyextproc.ProcessingResponse, error) { + switch r := req.Request.(type) { + case *envoyextproc.ProcessingRequest_RequestHeaders, *envoyextproc.ProcessingRequest_ResponseHeaders: + return nil, nil + + case *envoyextproc.ProcessingRequest_RequestBody: + // TODO: Handle request raw body in the WAF + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestBody{ + RequestBody: &envoyextproc.BodyResponse{ + Response: &envoyextproc.CommonResponse{ + Status: envoyextproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil + + case *envoyextproc.ProcessingRequest_RequestTrailers: + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestTrailers{}, + }, nil + + case *envoyextproc.ProcessingRequest_ResponseBody: + // Note: The end of stream bool value is not reliable + // Sometimes it's not set to true even if there is no more data to process + if r.ResponseBody.GetEndOfStream() { + return nil, nil + } + + // TODO: Handle response raw body in the WAF + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_ResponseBody{}, + }, nil + + case *envoyextproc.ProcessingRequest_ResponseTrailers: + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestTrailers{}, + }, nil + + default: + return nil, status.Errorf(codes.Unknown, "Unknown request type: %T", r) + } +} + +func processRequestHeaders(ctx context.Context, req *envoyextproc.ProcessingRequest_RequestHeaders) (*envoyextproc.ProcessingResponse, *currentRequest, bool, error) { + log.Debug("external_processing: received request headers: %v\n", req.RequestHeaders) + + request, err := newRequest(ctx, req) + if err != nil { + return nil, nil, false, status.Errorf(codes.InvalidArgument, "Error processing request headers from ext_proc: %v", err) + } + + var blocked bool + fakeResponseWriter := newFakeResponseWriter() + wrappedResponseWriter, request, afterHandle, blocked := httptrace.BeforeHandle(&httptrace.ServeConfig{ + SpanOpts: []ddtrace.StartSpanOption{ + tracer.Tag(ext.SpanKind, ext.SpanKindServer), + tracer.Tag(ext.Component, componentName), + }, + }, fakeResponseWriter, request) + + // Block handling: If triggered, we need to block the request, return an immediate response + if blocked { + afterHandle() + return doBlockResponse(fakeResponseWriter), nil, true, nil + } + + span, ok := tracer.SpanFromContext(request.Context()) + if !ok { + return nil, nil, false, status.Errorf(codes.Unknown, "Error getting span from context") + } + + processingResponse, err := propagationRequestHeaderMutation(span) + if err != nil { + return nil, nil, false, err + } + + return processingResponse, ¤tRequest{ + span: span, + ctx: request.Context(), + fakeResponseWriter: fakeResponseWriter, + wrappedResponseWriter: wrappedResponseWriter, + afterHandle: afterHandle, + }, false, nil +} + +func propagationRequestHeaderMutation(span ddtrace.Span) (*envoyextproc.ProcessingResponse, error) { + newHeaders := make(http.Header) + if err := tracer.Inject(span.Context(), tracer.HTTPHeadersCarrier(newHeaders)); err != nil { + return nil, status.Errorf(codes.Unknown, "Error injecting headers: %v", err) + } + + if len(newHeaders) > 0 { + log.Debug("external_processing: injecting propagation headers: %v\n", newHeaders) + } + + headerValueOptions := make([]*envoycore.HeaderValueOption, 0, len(newHeaders)) + for k, v := range newHeaders { + headerValueOptions = append(headerValueOptions, &envoycore.HeaderValueOption{ + Header: &envoycore.HeaderValue{ + Key: k, + RawValue: []byte(strings.Join(v, ",")), + }, + }) + } + + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_RequestHeaders{ + RequestHeaders: &envoyextproc.HeadersResponse{ + Response: &envoyextproc.CommonResponse{ + Status: envoyextproc.CommonResponse_CONTINUE, + HeaderMutation: &envoyextproc.HeaderMutation{ + SetHeaders: headerValueOptions, + }, + }, + }, + }, + }, nil +} + +func processResponseHeaders(res *envoyextproc.ProcessingRequest_ResponseHeaders, currentRequest *currentRequest) (*envoyextproc.ProcessingResponse, error) { + log.Debug("external_processing: received response headers: %v\n", res.ResponseHeaders) + + if err := createFakeResponseWriter(currentRequest.wrappedResponseWriter, res); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Error processing response headers from ext_proc: %v", err) + } + + var blocked bool + + // Now we need to know if the request has been blocked, but we don't have any other way than to look for the operation and bind a blocking data listener to it + op, ok := dyngo.FromContext(currentRequest.ctx) + if ok { + dyngo.OnData(op, func(_ *actions.BlockHTTP) { + // We already wrote over the response writer, we need to reset it so the blocking handler can write to it + httptrace.ResetStatusCode(currentRequest.wrappedResponseWriter) + currentRequest.fakeResponseWriter.Reset() + blocked = true + }) + } + + currentRequest.afterHandle() + + if blocked { + response := doBlockResponse(currentRequest.fakeResponseWriter) + return response, nil + } + + log.Debug("external_processing: finishing request with status code: %v\n", currentRequest.fakeResponseWriter.status) + + // Note: (cf. comment in the stream error handling) + // The end of stream bool value is not reliable + if res.ResponseHeaders.GetEndOfStream() { + return nil, nil + } + + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_ResponseHeaders{ + ResponseHeaders: &envoyextproc.HeadersResponse{ + Response: &envoyextproc.CommonResponse{ + Status: envoyextproc.CommonResponse_CONTINUE, + }, + }, + }, + }, nil +} + +func doBlockResponse(writer *fakeResponseWriter) *envoyextproc.ProcessingResponse { + var headersMutation []*envoycore.HeaderValueOption + for k, v := range writer.headers { + headersMutation = append(headersMutation, &envoycore.HeaderValueOption{ + Header: &envoycore.HeaderValue{ + Key: k, + RawValue: []byte(strings.Join(v, ",")), + }, + }) + } + + var int32StatusCode int32 = 0 + if writer.status > 0 && writer.status <= math.MaxInt32 { + int32StatusCode = int32(writer.status) + } + + return &envoyextproc.ProcessingResponse{ + Response: &envoyextproc.ProcessingResponse_ImmediateResponse{ + ImmediateResponse: &envoyextproc.ImmediateResponse{ + Status: &envoytypes.HttpStatus{ + Code: envoytypes.StatusCode(int32StatusCode), + }, + Headers: &envoyextproc.HeaderMutation{ + SetHeaders: headersMutation, + }, + Body: string(writer.body), + GrpcStatus: &envoyextproc.GrpcStatus{ + Status: 0, + }, + }, + }, + } +} diff --git a/contrib/envoyproxy/go-control-plane/envoy_test.go b/contrib/envoyproxy/go-control-plane/envoy_test.go new file mode 100644 index 0000000000..8af05eaab3 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/envoy_test.go @@ -0,0 +1,576 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "testing" + + envoyextproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + envoytypes "github.com/envoyproxy/go-control-plane/envoy/type/v3" + + ddgrpc "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" + + v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestAppSec(t *testing.T) { + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("monitoring-event-on-request", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "GET", map[string]string{"User-Agent": "dd-test-scanner-log"}, map[string]string{}, false) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"ua0-600-55x": 1}) + }) + + t.Run("blocking-event-on-request", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"User-Agent": "dd-test-scanner-log-block"}, "GET", "/"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(403), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"ua0-600-56x": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func TestBlockingWithUserRulesFile(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/user_rules.json") + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("blocking-event-on-response", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "OPTION", map[string]string{"User-Agent": "dd-test-scanner-log-block"}, map[string]string{"User-Agent": "match-response-headers"}, true) + + // Handle the immediate response + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) // 418 because of the rule file + require.Len(t, res.GetImmediateResponse().GetHeaders().SetHeaders, 1) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"headers-003": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) + + t.Run("blocking-event-on-request-on-query", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"User-Agent": "Mistake Not..."}, "GET", "/hello?match=match-request-query"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"query-002": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) + + t.Run("blocking-event-on-request-on-cookies", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"Cookie": "foo=jdfoSDGFkivRG_234"}, "OPTIONS", "/"), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(418), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"tst-037-008": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func TestGeneratedSpan(t *testing.T) { + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("request-span", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/resource-span", "GET", map[string]string{"user-agent": "Mistake Not...", "test-key": "test-value"}, map[string]string{"response-test-key": "response-test-value"}, false) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + + // Check for tags + span := finished[0] + require.Equal(t, "http.request", span.OperationName()) + require.Equal(t, "https://datadoghq.com/resource-span", span.Tag("http.url")) + require.Equal(t, "GET", span.Tag("http.method")) + require.Equal(t, "datadoghq.com", span.Tag("http.host")) + // require.Equal(t, "GET /resource-span", span.Tag("resource.name")) + require.Equal(t, "server", span.Tag("span.kind")) + require.Equal(t, "Mistake Not...", span.Tag("http.useragent")) + }) +} + +func TestXForwardedForHeaderClientIp(t *testing.T) { + t.Setenv("DD_APPSEC_RULES", "../../../internal/appsec/testdata/blocking.json") + appsec.Start() + defer appsec.Stop() + if !appsec.Enabled() { + t.Skip("appsec disabled") + } + + setup := func() (envoyextproc.ExternalProcessorClient, mocktracer.Tracer, func()) { + rig, err := newEnvoyAppsecRig(t, false) + require.NoError(t, err) + + mt := mocktracer.Start() + + return rig.client, mt, func() { + rig.Close() + mt.Stop() + } + } + + t.Run("client-ip", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + end2EndStreamRequest(t, stream, "/", "OPTION", + map[string]string{"User-Agent": "Mistake not...", "X-Forwarded-For": "18.18.18.18"}, + map[string]string{"User-Agent": "match-response-headers"}, + true) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + + // Check for tags + span := finished[0] + require.Equal(t, "18.18.18.18", span.Tag("http.client_ip")) + + // Appsec + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + }) + + t.Run("blocking-client-ip", func(t *testing.T) { + client, mt, cleanup := setup() + defer cleanup() + + ctx := context.Background() + stream, err := client.Process(ctx) + require.NoError(t, err) + + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, map[string]string{"User-Agent": "Mistake not...", "X-Forwarded-For": "1.2.3.4"}, "GET", "/"), + }, + }, + }) + require.NoError(t, err) + + // Handle the immediate response + res, err := stream.Recv() + require.Equal(t, uint32(0), res.GetImmediateResponse().GetGrpcStatus().Status) + require.Equal(t, envoytypes.StatusCode(403), res.GetImmediateResponse().GetStatus().Code) + require.Equal(t, "Content-Type", res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().Key) + require.Equal(t, "application/json", string(res.GetImmediateResponse().GetHeaders().SetHeaders[0].GetHeader().RawValue)) + require.NoError(t, err) + + err = stream.CloseSend() + require.NoError(t, err) + stream.Recv() // to flush the spans + + finished := mt.FinishedSpans() + require.Len(t, finished, 1) + checkForAppsecEvent(t, finished, map[string]int{"blk-001-001": 1}) + + // Check for tags + span := finished[0] + require.Equal(t, "1.2.3.4", span.Tag("http.client_ip")) + require.Equal(t, 1, span.Tag("_dd.appsec.enabled")) + require.Equal(t, true, span.Tag("appsec.event")) + require.Equal(t, true, span.Tag("appsec.blocked")) + }) +} + +func newEnvoyAppsecRig(t *testing.T, traceClient bool, interceptorOpts ...ddgrpc.Option) (*envoyAppsecRig, error) { + t.Helper() + + interceptorOpts = append([]ddgrpc.InterceptorOption{ddgrpc.WithServiceName("grpc")}, interceptorOpts...) + + server := grpc.NewServer() + + fixtureServer := new(envoyFixtureServer) + appsecSrv := AppsecEnvoyExternalProcessorServer(fixtureServer) + envoyextproc.RegisterExternalProcessorServer(server, appsecSrv) + + li, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + _, port, _ := net.SplitHostPort(li.Addr().String()) + // start our test fixtureServer. + go server.Serve(li) + + opts := []grpc.DialOption{grpc.WithInsecure()} + if traceClient { + opts = append(opts, + grpc.WithStreamInterceptor(ddgrpc.StreamClientInterceptor(interceptorOpts...)), + ) + } + conn, err := grpc.Dial(li.Addr().String(), opts...) + if err != nil { + return nil, fmt.Errorf("error dialing: %s", err) + } + return &envoyAppsecRig{ + fixtureServer: fixtureServer, + listener: li, + port: port, + server: server, + conn: conn, + client: envoyextproc.NewExternalProcessorClient(conn), + }, err +} + +// rig contains all servers and connections we'd need for a grpc integration test +type envoyAppsecRig struct { + fixtureServer *envoyFixtureServer + server *grpc.Server + port string + listener net.Listener + conn *grpc.ClientConn + client envoyextproc.ExternalProcessorClient +} + +func (r *envoyAppsecRig) Close() { + r.server.Stop() + r.conn.Close() +} + +type envoyFixtureServer struct { + envoyextproc.ExternalProcessorServer +} + +// Helper functions + +func end2EndStreamRequest(t *testing.T, stream envoyextproc.ExternalProcessor_ProcessClient, path string, method string, requestHeaders map[string]string, responseHeaders map[string]string, blockOnResponse bool) { + t.Helper() + + // First part: request + // 1- Send the headers + err := stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestHeaders{ + RequestHeaders: &envoyextproc.HttpHeaders{ + Headers: makeRequestHeaders(t, requestHeaders, method, path), + }, + }, + }) + require.NoError(t, err) + + res, err := stream.Recv() + require.NoError(t, err) + require.Equal(t, envoyextproc.CommonResponse_CONTINUE, res.GetRequestHeaders().GetResponse().GetStatus()) + + // 2- Send the body + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestBody{ + RequestBody: &envoyextproc.HttpBody{ + Body: []byte("body"), + }, + }, + }) + require.NoError(t, err) + + res, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, envoyextproc.CommonResponse_CONTINUE, res.GetRequestBody().GetResponse().GetStatus()) + + // 3- Send the trailers + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_RequestTrailers{ + RequestTrailers: &envoyextproc.HttpTrailers{ + Trailers: &v3.HeaderMap{ + Headers: []*v3.HeaderValue{ + {Key: "key", Value: "value"}, + }, + }, + }, + }, + }) + require.NoError(t, err) + + res, err = stream.Recv() + require.NoError(t, err) + require.NotNil(t, res.GetRequestTrailers()) + + // Second part: response + // 1- Send the response headers + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_ResponseHeaders{ + ResponseHeaders: &envoyextproc.HttpHeaders{ + Headers: makeResponseHeaders(t, responseHeaders, "200"), + }, + }, + }) + require.NoError(t, err) + + if blockOnResponse { + // Should have received an immediate response for blocking + // Let the test handle the response + return + } + + res, err = stream.Recv() + require.NoError(t, err) + require.Equal(t, envoyextproc.CommonResponse_CONTINUE, res.GetResponseHeaders().GetResponse().GetStatus()) + + // 2- Send the response body + err = stream.Send(&envoyextproc.ProcessingRequest{ + Request: &envoyextproc.ProcessingRequest_ResponseBody{ + ResponseBody: &envoyextproc.HttpBody{ + Body: []byte("body"), + EndOfStream: true, + }, + }, + }) + require.NoError(t, err) + + // The stream should now be closed + _, err = stream.Recv() + require.Equal(t, io.EOF, err) +} + +func checkForAppsecEvent(t *testing.T, finished []mocktracer.Span, expectedRuleIDs map[string]int) { + t.Helper() + + // The request should have the attack attempts + event := finished[len(finished)-1].Tag("_dd.appsec.json") + require.NotNil(t, event, "the _dd.appsec.json tag was not found") + + jsonText := event.(string) + type trigger struct { + Rule struct { + ID string `json:"id"` + } `json:"rule"` + } + var parsed struct { + Triggers []trigger `json:"triggers"` + } + err := json.Unmarshal([]byte(jsonText), &parsed) + require.NoError(t, err) + + histogram := map[string]uint8{} + for _, tr := range parsed.Triggers { + histogram[tr.Rule.ID]++ + } + + for ruleID, count := range expectedRuleIDs { + require.Equal(t, count, int(histogram[ruleID]), "rule %s has been triggered %d times but expected %d") + } + + require.Len(t, parsed.Triggers, len(expectedRuleIDs), "unexpected number of rules triggered") +} + +// Construct request headers +func makeRequestHeaders(t *testing.T, headers map[string]string, method string, path string) *v3.HeaderMap { + t.Helper() + + h := &v3.HeaderMap{} + for k, v := range headers { + h.Headers = append(h.Headers, &v3.HeaderValue{Key: k, RawValue: []byte(v)}) + } + + h.Headers = append(h.Headers, + &v3.HeaderValue{Key: ":method", RawValue: []byte(method)}, + &v3.HeaderValue{Key: ":path", RawValue: []byte(path)}, + &v3.HeaderValue{Key: ":scheme", RawValue: []byte("https")}, + &v3.HeaderValue{Key: ":authority", RawValue: []byte("datadoghq.com")}, + ) + + return h +} + +func makeResponseHeaders(t *testing.T, headers map[string]string, status string) *v3.HeaderMap { + t.Helper() + + h := &v3.HeaderMap{} + for k, v := range headers { + h.Headers = append(h.Headers, &v3.HeaderValue{Key: k, RawValue: []byte(v)}) + } + + h.Headers = append(h.Headers, &v3.HeaderValue{Key: ":status", RawValue: []byte(status)}) + + return h +} diff --git a/contrib/envoyproxy/go-control-plane/example_test.go b/contrib/envoyproxy/go-control-plane/example_test.go new file mode 100644 index 0000000000..f1e255dcaf --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/example_test.go @@ -0,0 +1,44 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane_test + +import ( + "log" + "net" + + "google.golang.org/grpc" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + gocontrolplane "gopkg.in/DataDog/dd-trace-go.v1/contrib/envoyproxy/go-control-plane" +) + +// interface fpr external processing server +type envoyExtProcServer struct { + extprocv3.ExternalProcessorServer +} + +func Example_server() { + // Create a listener for the server. + ln, err := net.Listen("tcp", ":50051") + if err != nil { + log.Fatal(err) + } + + // Initialize the grpc server as normal, using the envoy server interceptor. + s := grpc.NewServer() + srv := &envoyExtProcServer{} + + // Register the appsec envoy external processor service + appsecSrv := gocontrolplane.AppsecEnvoyExternalProcessorServer(srv) + extprocv3.RegisterExternalProcessorServer(s, appsecSrv) + + // ... register your services + + // Start serving incoming connections. + if err := s.Serve(ln); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/contrib/envoyproxy/go-control-plane/fakehttp.go b/contrib/envoyproxy/go-control-plane/fakehttp.go new file mode 100644 index 0000000000..3f20725e1b --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/fakehttp.go @@ -0,0 +1,189 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package go_control_plane + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "google.golang.org/grpc/metadata" +) + +// checkPseudoRequestHeaders Verify the required HTTP2 headers are present +// Some mandatory headers need to be set. It can happen when it wasn't a real HTTP2 request sent by Envoy, +func checkPseudoRequestHeaders(headers map[string]string) error { + for _, header := range []string{":authority", ":scheme", ":path", ":method"} { + if _, ok := headers[header]; !ok { + return fmt.Errorf("missing required headers: %q", header) + } + } + + return nil +} + +// checkPseudoResponseHeaders verifies the required HTTP2 headers are present +// Some mandatory headers need to be set. It can happen when it wasn't a real HTTP2 request sent by Envoy, +func checkPseudoResponseHeaders(headers map[string]string) error { + if _, ok := headers[":status"]; !ok { + return fmt.Errorf("missing required ':status' headers") + } + + return nil +} + +func getRemoteAddr(md metadata.MD) string { + xfwd := md.Get("x-forwarded-for") + length := len(xfwd) + if length == 0 { + return "" + } + + // Get the first right value of x-forwarded-for headers + // The rightmost IP address is the one that will be used as the remote client IP + // https://datadoghq.atlassian.net/wiki/spaces/TS/pages/2766733526/Sensitive+IP+information#Where-does-the-value-of-the-http.client_ip-tag-come-from%3F + return xfwd[length-1] +} + +// splitPseudoHeaders splits normal headers of the initial request made by the client and the pseudo headers of HTTP/2 +// - Format the headers to be used by the tracer as a map[string][]string +// - Set headers keys to be canonical +func splitPseudoHeaders(receivedHeaders []*corev3.HeaderValue) (headers map[string][]string, pseudoHeaders map[string]string) { + headers = make(map[string][]string, len(receivedHeaders)-4) + pseudoHeaders = make(map[string]string, 4) + for _, v := range receivedHeaders { + key := v.GetKey() + if key == "" { + continue + } + if key[0] == ':' { + pseudoHeaders[key] = string(v.GetRawValue()) + continue + } + + headers[http.CanonicalHeaderKey(key)] = []string{string(v.GetRawValue())} + } + return headers, pseudoHeaders +} + +func createFakeResponseWriter(w http.ResponseWriter, res *extproc.ProcessingRequest_ResponseHeaders) error { + headers, pseudoHeaders := splitPseudoHeaders(res.ResponseHeaders.GetHeaders().GetHeaders()) + + if err := checkPseudoResponseHeaders(pseudoHeaders); err != nil { + return err + } + + status, err := strconv.Atoi(pseudoHeaders[":status"]) + if err != nil { + return fmt.Errorf("error parsing status code %q: %w", pseudoHeaders[":status"], err) + } + + for k, v := range headers { + w.Header().Set(k, strings.Join(v, ",")) + } + + w.WriteHeader(status) + return nil +} + +// newRequest creates a new http.Request from an ext_proc RequestHeaders message +func newRequest(ctx context.Context, req *extproc.ProcessingRequest_RequestHeaders) (*http.Request, error) { + headers, pseudoHeaders := splitPseudoHeaders(req.RequestHeaders.GetHeaders().GetHeaders()) + if err := checkPseudoRequestHeaders(pseudoHeaders); err != nil { + return nil, err + } + + parsedURL, err := url.Parse(fmt.Sprintf("%s://%s%s", pseudoHeaders[":scheme"], pseudoHeaders[":authority"], pseudoHeaders[":path"])) + if err != nil { + return nil, fmt.Errorf( + "error building envoy URI from scheme %q, from host %q and from path %q: %w", + pseudoHeaders[":scheme"], + pseudoHeaders[":host"], + pseudoHeaders[":path"], + err) + } + + var remoteAddr string + md, ok := metadata.FromIncomingContext(ctx) + if ok { + remoteAddr = getRemoteAddr(md) + } + + var tlsState *tls.ConnectionState + if pseudoHeaders[":scheme"] == "https" { + tlsState = &tls.ConnectionState{} + } + + headers["Host"] = append(headers["Host"], pseudoHeaders[":authority"]) + + return (&http.Request{ + Method: pseudoHeaders[":method"], + Host: pseudoHeaders[":authority"], + RequestURI: pseudoHeaders[":path"], + URL: parsedURL, + Header: headers, + RemoteAddr: remoteAddr, + TLS: tlsState, + }).WithContext(ctx), nil +} + +type fakeResponseWriter struct { + mu sync.Mutex + status int + body []byte + headers http.Header +} + +// Reset resets the fakeResponseWriter to its initial state +func (w *fakeResponseWriter) Reset() { + w.mu.Lock() + defer w.mu.Unlock() + w.status = 0 + w.body = nil + w.headers = make(http.Header) +} + +// Status is not in the [http.ResponseWriter] interface, but it is cast into it by the tracing code +func (w *fakeResponseWriter) Status() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.status +} + +func (w *fakeResponseWriter) WriteHeader(status int) { + w.mu.Lock() + defer w.mu.Unlock() + w.status = status +} + +func (w *fakeResponseWriter) Header() http.Header { + w.mu.Lock() + defer w.mu.Unlock() + return w.headers +} + +func (w *fakeResponseWriter) Write(b []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.body = append(w.body, b...) + return len(b), nil +} + +var _ http.ResponseWriter = &fakeResponseWriter{} + +// newFakeResponseWriter creates a new fakeResponseWriter that can be used to store the response a [http.Handler] made +func newFakeResponseWriter() *fakeResponseWriter { + return &fakeResponseWriter{ + headers: make(http.Header), + } +} diff --git a/contrib/gofiber/fiber.v2/fiber.go b/contrib/gofiber/fiber.v2/fiber.go index 4181354290..206ce22502 100644 --- a/contrib/gofiber/fiber.v2/fiber.go +++ b/contrib/gofiber/fiber.v2/fiber.go @@ -62,6 +62,10 @@ func Middleware(opts ...Option) func(c *fiber.Ctx) error { } } if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(h)); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } opts = append(opts, cfg.spanOpts...) diff --git a/contrib/google.golang.org/api/api_test.go b/contrib/google.golang.org/api/api_test.go index a9b8f54347..d5104b9690 100644 --- a/contrib/google.golang.org/api/api_test.go +++ b/contrib/google.golang.org/api/api_test.go @@ -63,7 +63,7 @@ func TestBooks(t *testing.T) { assert.Equal(t, "books.bookshelves.list", s0.Tag(ext.ResourceName)) assert.Equal(t, "400", s0.Tag(ext.HTTPCode)) assert.Equal(t, "GET", s0.Tag(ext.HTTPMethod)) - assert.Equal(t, svc.BasePath+"books/v1/users/montana.banana/bookshelves?alt=json&prettyPrint=false", s0.Tag(ext.HTTPURL)) + assert.Equal(t, svc.BasePath+"books/v1/users/montana.banana/bookshelves", s0.Tag(ext.HTTPURL)) assert.Equal(t, "google.golang.org/api", s0.Tag(ext.Component)) assert.Equal(t, ext.SpanKindClient, s0.Tag(ext.SpanKind)) } @@ -88,7 +88,7 @@ func TestCivicInfo(t *testing.T) { assert.Equal(t, "civicinfo.representatives.representativeInfoByAddress", s0.Tag(ext.ResourceName)) assert.Equal(t, "400", s0.Tag(ext.HTTPCode)) assert.Equal(t, "GET", s0.Tag(ext.HTTPMethod)) - assert.Equal(t, svc.BasePath+"civicinfo/v2/representatives?alt=json&prettyPrint=false", s0.Tag(ext.HTTPURL)) + assert.Equal(t, svc.BasePath+"civicinfo/v2/representatives", s0.Tag(ext.HTTPURL)) assert.Equal(t, "google.golang.org/api", s0.Tag(ext.Component)) assert.Equal(t, ext.SpanKindClient, s0.Tag(ext.SpanKind)) } @@ -115,7 +115,7 @@ func TestURLShortener(t *testing.T) { assert.Equal(t, "urlshortener.url.list", s0.Tag(ext.ResourceName)) assert.Equal(t, "400", s0.Tag(ext.HTTPCode)) assert.Equal(t, "GET", s0.Tag(ext.HTTPMethod)) - assert.Equal(t, "https://www.googleapis.com/urlshortener/v1/url/history?alt=json&prettyPrint=false", s0.Tag(ext.HTTPURL)) + assert.Equal(t, "https://www.googleapis.com/urlshortener/v1/url/history", s0.Tag(ext.HTTPURL)) assert.Equal(t, "google.golang.org/api", s0.Tag(ext.Component)) assert.Equal(t, ext.SpanKindClient, s0.Tag(ext.SpanKind)) } @@ -140,7 +140,7 @@ func TestWithEndpointMetadataDisabled(t *testing.T) { assert.Equal(t, "GET civicinfo.googleapis.com", s0.Tag(ext.ResourceName)) assert.Equal(t, "400", s0.Tag(ext.HTTPCode)) assert.Equal(t, "GET", s0.Tag(ext.HTTPMethod)) - assert.Equal(t, svc.BasePath+"civicinfo/v2/representatives?alt=json&prettyPrint=false", s0.Tag(ext.HTTPURL)) + assert.Equal(t, svc.BasePath+"civicinfo/v2/representatives", s0.Tag(ext.HTTPURL)) assert.Equal(t, "google.golang.org/api", s0.Tag(ext.Component)) assert.Equal(t, ext.SpanKindClient, s0.Tag(ext.SpanKind)) } diff --git a/contrib/google.golang.org/grpc/grpc.go b/contrib/google.golang.org/grpc/grpc.go index 323edbb542..24ca5b7d61 100644 --- a/contrib/google.golang.org/grpc/grpc.go +++ b/contrib/google.golang.org/grpc/grpc.go @@ -71,6 +71,10 @@ func startSpanFromContext( ) md, _ := metadata.FromIncomingContext(ctx) // nil is ok if sctx, err := tracer.Extract(grpcutil.MDCarrier(md)); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := sctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(sctx)) } return tracer.StartSpanFromContext(ctx, operation, opts...) diff --git a/contrib/internal/httptrace/httptrace.go b/contrib/internal/httptrace/httptrace.go index 92069e03d8..4aad5aba38 100644 --- a/contrib/internal/httptrace/httptrace.go +++ b/contrib/internal/httptrace/httptrace.go @@ -51,7 +51,11 @@ func StartRequestSpan(r *http.Request, opts ...ddtrace.StartSpanOption) (tracer. cfg.Tags["http.host"] = r.Host } if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(r.Header)); err == nil { - cfg.Parent = spanctx + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + tracer.WithSpanLinks(linksCtx.SpanLinks())(cfg) + } + tracer.ChildOf(spanctx)(cfg) } for k, v := range ipTags { cfg.Tags[k] = v diff --git a/contrib/internal/httptrace/response_writer.go b/contrib/internal/httptrace/response_writer.go index 2bbc31bad7..f44fff762f 100644 --- a/contrib/internal/httptrace/response_writer.go +++ b/contrib/internal/httptrace/response_writer.go @@ -16,6 +16,13 @@ type responseWriter struct { status int } +// ResetStatusCode resets the status code of the response writer. +func ResetStatusCode(w http.ResponseWriter) { + if rw, ok := w.(*responseWriter); ok { + rw.status = 0 + } +} + func newResponseWriter(w http.ResponseWriter) *responseWriter { return &responseWriter{w, 0} } diff --git a/contrib/log/slog/slog_test.go b/contrib/log/slog/slog_test.go index 0d25702cb2..7546009b0f 100644 --- a/contrib/log/slog/slog_test.go +++ b/contrib/log/slog/slog_test.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "testing" + "testing/slogtest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -84,13 +85,33 @@ func TestNewJSONHandler(t *testing.T) { } func TestWrapHandler(t *testing.T) { - testLogger( - t, - func(w io.Writer) *slog.Logger { - return slog.New(WrapHandler(slog.NewJSONHandler(w, nil))) - }, - nil, - ) + t.Run("testLogger", func(t *testing.T) { + testLogger( + t, + func(w io.Writer) *slog.Logger { + return slog.New(WrapHandler(slog.NewJSONHandler(w, nil))) + }, + nil, + ) + }) + + t.Run("slogtest", func(t *testing.T) { + var buf bytes.Buffer + h := WrapHandler(slog.NewJSONHandler(&buf, nil)) + results := func() []map[string]any { + var ms []map[string]any + for _, line := range bytes.Split(buf.Bytes(), []byte{'\n'}) { + if len(line) == 0 { + continue + } + var m map[string]any + require.NoError(t, json.Unmarshal(line, &m)) + ms = append(ms, m) + } + return ms + } + require.NoError(t, slogtest.TestHandler(h, results)) + }) } func TestHandlerWithAttrs(t *testing.T) { diff --git a/contrib/net/http/option.go b/contrib/net/http/option.go index 22e63ee3b0..79144dc69c 100644 --- a/contrib/net/http/option.go +++ b/contrib/net/http/option.go @@ -174,7 +174,7 @@ func newRoundTripperConfig() *roundTripperConfig { propagation: true, spanNamer: defaultSpanNamer, ignoreRequest: func(_ *http.Request) bool { return false }, - queryString: internal.BoolEnv(envClientQueryStringEnabled, true), + queryString: internal.BoolEnv(envClientQueryStringEnabled, false), isStatusError: isClientError, } v := os.Getenv(envClientErrorStatuses) diff --git a/contrib/net/http/roundtripper_test.go b/contrib/net/http/roundtripper_test.go index 4002647fe6..a15904fb7f 100644 --- a/contrib/net/http/roundtripper_test.go +++ b/contrib/net/http/roundtripper_test.go @@ -579,19 +579,19 @@ func TestClientQueryString(t *testing.T) { client := &http.Client{ Transport: rt, } - resp, err := client.Get(s.URL + "/hello/world?querystring=xyz") + resp, err := client.Get(s.URL + "/hello/world?API_KEY=1234") assert.Nil(t, err) defer resp.Body.Close() spans := mt.FinishedSpans() assert.Len(t, spans, 1) - assert.Regexp(t, regexp.MustCompile(`^http://.*?/hello/world\?querystring=xyz$`), spans[0].Tag(ext.HTTPURL)) + assert.Regexp(t, regexp.MustCompile(`^http://.*?/hello/world$`), spans[0].Tag(ext.HTTPURL)) }) - t.Run("false", func(t *testing.T) { + t.Run("true", func(t *testing.T) { mt := mocktracer.Start() defer mt.Stop() - os.Setenv("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING", "false") + os.Setenv("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING", "true") defer os.Unsetenv("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING") rt := WrapRoundTripper(http.DefaultTransport) @@ -604,29 +604,27 @@ func TestClientQueryString(t *testing.T) { spans := mt.FinishedSpans() assert.Len(t, spans, 1) - assert.Regexp(t, regexp.MustCompile(`^http://.*?/hello/world$`), spans[0].Tag(ext.HTTPURL)) + assert.Regexp(t, regexp.MustCompile(`^http://.*?/hello/world\?querystring=xyz$`), spans[0].Tag(ext.HTTPURL)) }) // DD_TRACE_HTTP_URL_QUERY_STRING_DISABLED applies only to server spans, not client t.Run("Not impacted by DD_TRACE_HTTP_URL_QUERY_STRING_DISABLED", func(t *testing.T) { mt := mocktracer.Start() defer mt.Stop() - os.Setenv("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING", "true") - os.Setenv("DD_TRACE_HTTP_URL_QUERY_STRING_DISABLED", "true") - defer os.Unsetenv("DD_TRACE_HTTP_CLIENT_TAG_QUERY_STRING") + os.Setenv("DD_TRACE_HTTP_URL_QUERY_STRING_DISABLED", "false") defer os.Unsetenv("DD_TRACE_HTTP_URL_QUERY_STRING_DISABLED") rt := WrapRoundTripper(http.DefaultTransport) client := &http.Client{ Transport: rt, } - resp, err := client.Get(s.URL + "/hello/world?querystring=xyz") + resp, err := client.Get(s.URL + "/hello/world?API_KEY=1234") assert.Nil(t, err) defer resp.Body.Close() spans := mt.FinishedSpans() assert.Len(t, spans, 1) - assert.Contains(t, spans[0].Tag(ext.HTTPURL), "/hello/world?querystring=xyz") + assert.Regexp(t, regexp.MustCompile(`^http://.*?/hello/world$`), spans[0].Tag(ext.HTTPURL)) }) } diff --git a/contrib/segmentio/kafka.go.v0/internal/tracing/tracing.go b/contrib/segmentio/kafka.go.v0/internal/tracing/tracing.go index 0fa9c5d052..824891ac7b 100644 --- a/contrib/segmentio/kafka.go.v0/internal/tracing/tracing.go +++ b/contrib/segmentio/kafka.go.v0/internal/tracing/tracing.go @@ -49,6 +49,10 @@ func (tr *Tracer) StartConsumeSpan(ctx context.Context, msg Message) ddtrace.Spa // kafka supports headers, so try to extract a span context carrier := NewMessageCarrier(msg) if spanctx, err := tracer.Extract(carrier); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } span, _ := tracer.StartSpanFromContext(ctx, tr.consumerSpanName, opts...) diff --git a/contrib/twitchtv/twirp/twirp.go b/contrib/twitchtv/twirp/twirp.go index 2912e722e2..b99510c1b9 100644 --- a/contrib/twitchtv/twirp/twirp.go +++ b/contrib/twitchtv/twirp/twirp.go @@ -14,6 +14,7 @@ import ( "net/http" "strconv" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" @@ -90,6 +91,10 @@ func (wc *wrappedClient) Do(req *http.Request) (*http.Response, error) { opts = append(opts, tracer.Tag(ext.EventSampleRate, wc.cfg.analyticsRate)) } if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(req.Header)); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + opts = append(opts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } opts = append(opts, tracer.ChildOf(spanctx)) } @@ -139,6 +144,10 @@ func WrapServer(h http.Handler, opts ...Option) http.Handler { spanOpts = append(spanOpts, tracer.Tag(ext.EventSampleRate, cfg.analyticsRate)) } if spanctx, err := tracer.Extract(tracer.HTTPHeadersCarrier(r.Header)); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := spanctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + spanOpts = append(spanOpts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } spanOpts = append(spanOpts, tracer.ChildOf(spanctx)) } span, ctx := tracer.StartSpanFromContext(r.Context(), "twirp.handler", spanOpts...) diff --git a/contrib/valyala/fasthttp.v1/fasthttp.go b/contrib/valyala/fasthttp.v1/fasthttp.go index dd2e2a5af7..c3d1bb4c3e 100644 --- a/contrib/valyala/fasthttp.v1/fasthttp.go +++ b/contrib/valyala/fasthttp.v1/fasthttp.go @@ -46,6 +46,10 @@ func WrapHandler(h fasthttp.RequestHandler, opts ...Option) fasthttp.RequestHand ReqHeader: &fctx.Request.Header, } if sctx, err := tracer.Extract(fcc); err == nil { + // If there are span links as a result of context extraction, add them as a StartSpanOption + if linksCtx, ok := sctx.(ddtrace.SpanContextWithLinks); ok && linksCtx.SpanLinks() != nil { + spanOpts = append(spanOpts, tracer.WithSpanLinks(linksCtx.SpanLinks())) + } spanOpts = append(spanOpts, tracer.ChildOf(sctx)) } span := fasthttptrace.StartSpanFromContext(fctx, "http.request", spanOpts...) diff --git a/ddtrace/mocktracer/mocktracer.go b/ddtrace/mocktracer/mocktracer.go index 2a210e07bb..396d99bf20 100644 --- a/ddtrace/mocktracer/mocktracer.go +++ b/ddtrace/mocktracer/mocktracer.go @@ -80,7 +80,7 @@ func newMockTracer() *mocktracer { client := &http.Client{ Transport: t.dsmTransport, } - t.dsmProcessor = datastreams.NewProcessor(&statsd.NoOpClient{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client) + t.dsmProcessor = datastreams.NewProcessor(&statsd.NoOpClientDirect{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client) t.dsmProcessor.Start() t.dsmProcessor.Flush() return &t diff --git a/ddtrace/tracer/civisibility_tslv_msgp.go b/ddtrace/tracer/civisibility_tslv_msgp.go index 63fa4b8499..37c4f06a52 100644 --- a/ddtrace/tracer/civisibility_tslv_msgp.go +++ b/ddtrace/tracer/civisibility_tslv_msgp.go @@ -497,25 +497,25 @@ func (z *tslvSpan) DecodeMsg(dc *msgp.Reader) (err error) { case "test_session_id": z.SessionID, err = dc.ReadUint64() if err != nil { - err = msgp.WrapError(err, "SessionId") + err = msgp.WrapError(err, "SessionID") return } case "test_module_id": z.ModuleID, err = dc.ReadUint64() if err != nil { - err = msgp.WrapError(err, "ModuleId") + err = msgp.WrapError(err, "ModuleID") return } case "test_suite_id": z.SuiteID, err = dc.ReadUint64() if err != nil { - err = msgp.WrapError(err, "SuiteId") + err = msgp.WrapError(err, "SuiteID") return } case "itr_correlation_id": z.CorrelationID, err = dc.ReadString() if err != nil { - err = msgp.WrapError(err, "CorrelationId") + err = msgp.WrapError(err, "CorrelationID") return } case "name": @@ -651,7 +651,7 @@ func (z *tslvSpan) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(16) var zb0001Mask uint16 /* 16 bits */ _ = zb0001Mask @@ -699,7 +699,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { if zb0001Len == 0 { return } - if (zb0001Mask & 0x1) == 0 { // if not empty + if (zb0001Mask & 0x1) == 0 { // if not omitted // write "test_session_id" err = en.Append(0xaf, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64) if err != nil { @@ -711,7 +711,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x2) == 0 { // if not empty + if (zb0001Mask & 0x2) == 0 { // if not omitted // write "test_module_id" err = en.Append(0xae, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x69, 0x64) if err != nil { @@ -723,7 +723,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x4) == 0 { // if not empty + if (zb0001Mask & 0x4) == 0 { // if not omitted // write "test_suite_id" err = en.Append(0xad, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x75, 0x69, 0x74, 0x65, 0x5f, 0x69, 0x64) if err != nil { @@ -735,7 +735,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x8) == 0 { // if not empty + if (zb0001Mask & 0x8) == 0 { // if not omitted // write "itr_correlation_id" err = en.Append(0xb2, 0x69, 0x74, 0x72, 0x5f, 0x63, 0x6f, 0x72, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64) if err != nil { @@ -807,7 +807,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Duration") return } - if (zb0001Mask & 0x400) == 0 { // if not empty + if (zb0001Mask & 0x400) == 0 { // if not omitted // write "span_id" err = en.Append(0xa7, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) if err != nil { @@ -819,7 +819,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x800) == 0 { // if not empty + if (zb0001Mask & 0x800) == 0 { // if not omitted // write "trace_id" err = en.Append(0xa8, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64) if err != nil { @@ -831,7 +831,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x1000) == 0 { // if not empty + if (zb0001Mask & 0x1000) == 0 { // if not omitted // write "parent_id" err = en.Append(0xa9, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64) if err != nil { @@ -853,7 +853,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Error") return } - if (zb0001Mask & 0x4000) == 0 { // if not empty + if (zb0001Mask & 0x4000) == 0 { // if not omitted // write "meta" err = en.Append(0xa4, 0x6d, 0x65, 0x74, 0x61) if err != nil { @@ -877,7 +877,7 @@ func (z *tslvSpan) EncodeMsg(en *msgp.Writer) (err error) { } } } - if (zb0001Mask & 0x8000) == 0 { // if not empty + if (zb0001Mask & 0x8000) == 0 { // if not omitted // write "metrics" err = en.Append(0xa7, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) if err != nil { diff --git a/ddtrace/tracer/civisibility_writer.go b/ddtrace/tracer/civisibility_writer.go index 65d671e97b..914a7ba1fa 100644 --- a/ddtrace/tracer/civisibility_writer.go +++ b/ddtrace/tracer/civisibility_writer.go @@ -122,7 +122,7 @@ func (w *ciVisibilityTraceWriter) flush() { } log.Errorf("ciVisibilityTraceWriter: failure sending events (attempt %d), will retry: %v", attempt+1, err) p.reset() - time.Sleep(time.Millisecond) + time.Sleep(w.config.retryInterval) } log.Errorf("ciVisibilityTraceWriter: lost %d events: %v", count, err) telemetry.EndpointPayloadDropped(telemetry.TestCycleEndpointType) diff --git a/ddtrace/tracer/civisibility_writer_test.go b/ddtrace/tracer/civisibility_writer_test.go index 5980ebb3e0..c12093f0d0 100644 --- a/ddtrace/tracer/civisibility_writer_test.go +++ b/ddtrace/tracer/civisibility_writer_test.go @@ -11,6 +11,7 @@ import ( "io" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/tinylib/msgp/msgp" @@ -57,21 +58,25 @@ func (t *failingCiVisibilityTransport) send(p *payload) (io.ReadCloser, error) { func TestCiVisibilityTraceWriterFlushRetries(t *testing.T) { testcases := []struct { configRetries int + retryInterval time.Duration failCount int tracesSent bool expAttempts int }{ - {configRetries: 0, failCount: 0, tracesSent: true, expAttempts: 1}, - {configRetries: 0, failCount: 1, tracesSent: false, expAttempts: 1}, + {configRetries: 0, retryInterval: time.Millisecond, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 0, retryInterval: time.Millisecond, failCount: 1, tracesSent: false, expAttempts: 1}, - {configRetries: 1, failCount: 0, tracesSent: true, expAttempts: 1}, - {configRetries: 1, failCount: 1, tracesSent: true, expAttempts: 2}, - {configRetries: 1, failCount: 2, tracesSent: false, expAttempts: 2}, + {configRetries: 1, retryInterval: time.Millisecond, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 1, retryInterval: time.Millisecond, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 1, retryInterval: time.Millisecond, failCount: 2, tracesSent: false, expAttempts: 2}, - {configRetries: 2, failCount: 0, tracesSent: true, expAttempts: 1}, - {configRetries: 2, failCount: 1, tracesSent: true, expAttempts: 2}, - {configRetries: 2, failCount: 2, tracesSent: true, expAttempts: 3}, - {configRetries: 2, failCount: 3, tracesSent: false, expAttempts: 3}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 2, tracesSent: true, expAttempts: 3}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 3, tracesSent: false, expAttempts: 3}, + + {configRetries: 1, retryInterval: 2 * time.Millisecond, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 2, retryInterval: 2 * time.Millisecond, failCount: 2, tracesSent: true, expAttempts: 3}, } ss := []*span{makeSpan(0)} @@ -86,16 +91,23 @@ func TestCiVisibilityTraceWriterFlushRetries(t *testing.T) { c := newConfig(func(c *config) { c.transport = p c.sendRetries = test.configRetries + c.retryInterval = test.retryInterval }) h := newCiVisibilityTraceWriter(c) h.add(ss) + start := time.Now() h.flush() h.wg.Wait() + elapsed := time.Since(start) assert.Equal(test.expAttempts, p.sendAttempts) assert.Equal(test.tracesSent, p.tracesSent) + + if test.configRetries > 0 && test.failCount > 1 { + assert.GreaterOrEqual(elapsed, test.retryInterval*time.Duration(minInts(test.configRetries+1, test.failCount))) + } }) } } diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 0ddf358992..21b5992396 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -136,10 +136,13 @@ type config struct { // output instead of using the agent. This is used in Lambda environments. logToStdout bool - // sendRetries is the number of times a trace payload send is retried upon + // sendRetries is the number of times a trace or CI Visibility payload send is retried upon // failure. sendRetries int + // retryInterval is the interval between agent connection retries. It has no effect if sendRetries is not set + retryInterval time.Duration + // logStartup, when true, causes various startup info to be written // when the tracer starts. logStartup bool @@ -456,7 +459,7 @@ func newConfig(opts ...StartOption) *config { if v := os.Getenv("DD_TRACE_PEER_SERVICE_MAPPING"); v != "" { internal.ForEachStringTag(v, internal.DDTagsDelimiter, func(key, val string) { c.peerServiceMappings[key] = val }) } - + c.retryInterval = time.Millisecond for _, fn := range opts { fn(c) } @@ -529,8 +532,19 @@ func newConfig(opts ...StartOption) *config { if c.debug { log.SetLevel(log.LevelDebug) } - // if using stdout or traces are disabled, agent is disabled - agentDisabled := c.logToStdout || !c.enabled.current + + // Check if CI Visibility mode is enabled + if internal.BoolEnv(constants.CIVisibilityEnabledEnvironmentVariable, false) { + c.ciVisibilityEnabled = true // Enable CI Visibility mode + c.httpClientTimeout = time.Second * 45 // Increase timeout up to 45 seconds (same as other tracers in CIVis mode) + c.logStartup = false // If we are in CI Visibility mode we don't want to log the startup to stdout to avoid polluting the output + ciTransport := newCiVisibilityTransport(c) // Create a default CI Visibility Transport + c.transport = ciTransport // Replace the default transport with the CI Visibility transport + c.ciVisibilityAgentless = ciTransport.agentless + } + + // if using stdout or traces are disabled or we are in ci visibility agentless mode, agent is disabled + agentDisabled := c.logToStdout || !c.enabled.current || c.ciVisibilityAgentless c.agent = loadAgentFeatures(agentDisabled, c.agentURL, c.httpClient) info, ok := debug.ReadBuildInfo() if !ok { @@ -548,17 +562,6 @@ func newConfig(opts ...StartOption) *config { // This allows persisting the initial value of globalTags for future resets and updates. globalTagsOrigin := c.globalTags.cfgOrigin c.initGlobalTags(c.globalTags.get(), globalTagsOrigin) - - // Check if CI Visibility mode is enabled - if internal.BoolEnv(constants.CIVisibilityEnabledEnvironmentVariable, false) { - c.ciVisibilityEnabled = true // Enable CI Visibility mode - c.httpClientTimeout = time.Second * 45 // Increase timeout up to 45 seconds (same as other tracers in CIVis mode) - c.logStartup = false // If we are in CI Visibility mode we don't want to log the startup to stdout to avoid polluting the output - ciTransport := newCiVisibilityTransport(c) // Create a default CI Visibility Transport - c.transport = ciTransport // Replace the default transport with the CI Visibility transport - c.ciVisibilityAgentless = ciTransport.agentless - } - return c } @@ -799,7 +802,7 @@ func statsTags(c *config) []string { // withNoopStats is used for testing to disable statsd client func withNoopStats() StartOption { return func(c *config) { - c.statsdClient = &statsd.NoOpClient{} + c.statsdClient = &statsd.NoOpClientDirect{} } } @@ -892,6 +895,13 @@ func WithSendRetries(retries int) StartOption { } } +// WithRetryInterval sets the interval, in seconds, for retrying submitting payloads to the agent. +func WithRetryInterval(interval int) StartOption { + return func(c *config) { + c.retryInterval = time.Duration(interval) * time.Second + } +} + // WithPropagator sets an alternative propagator to be used by the tracer. func WithPropagator(p Propagator) StartOption { return func(c *config) { diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 8e3a6262f7..ac96f1cdd3 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -855,6 +855,23 @@ func TestTracerOptionsDefaults(t *testing.T) { assert.Equal(t, time.Duration(10*time.Second), c.httpClient.Timeout) }) }) + + t.Run("trace-retries", func(t *testing.T) { + c := newConfig() + assert.Equal(t, 0, c.sendRetries) + assert.Equal(t, time.Millisecond, c.retryInterval) + }) +} + +func TestTraceRetry(t *testing.T) { + t.Run("sendRetries", func(t *testing.T) { + c := newConfig(WithSendRetries(10)) + assert.Equal(t, 10, c.sendRetries) + }) + t.Run("retryInterval", func(t *testing.T) { + c := newConfig(WithRetryInterval(10)) + assert.Equal(t, 10*time.Second, c.retryInterval) + }) } func TestDefaultHTTPClient(t *testing.T) { diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index f15da060ef..0297e48b53 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -78,7 +78,7 @@ type span struct { TraceID uint64 `msg:"trace_id"` // lower 64-bits of the root span identifier ParentID uint64 `msg:"parent_id"` // identifier of the span's direct parent Error int32 `msg:"error"` // error status of the span; 0 means no errors - SpanLinks []ddtrace.SpanLink `msg:"span_links"` // links to other spans + SpanLinks []ddtrace.SpanLink `msg:"span_links,omitempty"` // links to other spans goExecTraced bool `msg:"-"` noDebugStack bool `msg:"-"` // disables debug stack traces diff --git a/ddtrace/tracer/span_msgp.go b/ddtrace/tracer/span_msgp.go index c6cb8274a7..c4a65ef723 100644 --- a/ddtrace/tracer/span_msgp.go +++ b/ddtrace/tracer/span_msgp.go @@ -4,7 +4,6 @@ package tracer import ( "github.com/tinylib/msgp/msgp" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" ) @@ -273,7 +272,7 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *span) EncodeMsg(en *msgp.Writer) (err error) { - // omitempty: check for empty values + // check for omitted fields zb0001Len := uint32(14) var zb0001Mask uint16 /* 14 bits */ _ = zb0001Mask @@ -285,6 +284,10 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { zb0001Len-- zb0001Mask |= 0x100 } + if z.SpanLinks == nil { + zb0001Len-- + zb0001Mask |= 0x2000 + } // variable map header, size zb0001Len err = en.Append(0x80 | uint8(zb0001Len)) if err != nil { @@ -353,7 +356,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Duration") return } - if (zb0001Mask & 0x40) == 0 { // if not empty + if (zb0001Mask & 0x40) == 0 { // if not omitted // write "meta" err = en.Append(0xa4, 0x6d, 0x65, 0x74, 0x61) if err != nil { @@ -387,7 +390,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "MetaStruct") return } - if (zb0001Mask & 0x100) == 0 { // if not empty + if (zb0001Mask & 0x100) == 0 { // if not omitted // write "metrics" err = en.Append(0xa7, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) if err != nil { @@ -451,22 +454,24 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Error") return } - // write "span_links" - err = en.Append(0xaa, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x73) - if err != nil { - return - } - err = en.WriteArrayHeader(uint32(len(z.SpanLinks))) - if err != nil { - err = msgp.WrapError(err, "SpanLinks") - return - } - for za0005 := range z.SpanLinks { - err = z.SpanLinks[za0005].EncodeMsg(en) + if (zb0001Mask & 0x2000) == 0 { // if not omitted + // write "span_links" + err = en.Append(0xaa, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.SpanLinks))) if err != nil { - err = msgp.WrapError(err, "SpanLinks", za0005) + err = msgp.WrapError(err, "SpanLinks") return } + for za0005 := range z.SpanLinks { + err = z.SpanLinks[za0005].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "SpanLinks", za0005) + return + } + } } return } diff --git a/ddtrace/tracer/stats.go b/ddtrace/tracer/stats.go index 3b608e549f..28c8ff8bf3 100644 --- a/ddtrace/tracer/stats.go +++ b/ddtrace/tracer/stats.go @@ -54,7 +54,7 @@ type tracerStatSpan struct { // newConcentrator creates a new concentrator using the given tracer // configuration c. It creates buckets of bucketSize nanoseconds duration. -func newConcentrator(c *config, bucketSize int64) *concentrator { +func newConcentrator(c *config, bucketSize int64, statsdClient internal.StatsdClient) *concentrator { sCfg := &stats.SpanConcentratorConfig{ ComputeStatsBySpanKind: false, BucketInterval: defaultStatsBucketSize, @@ -86,6 +86,7 @@ func newConcentrator(c *config, bucketSize int64) *concentrator { cfg: c, aggregationKey: aggKey, spanConcentrator: spanConcentrator, + statsdClient: statsdClient, } } @@ -131,7 +132,7 @@ func (c *concentrator) runFlusher(tick <-chan time.Time) { // statsd returns any tracer configured statsd client, or a no-op. func (c *concentrator) statsd() internal.StatsdClient { if c.statsdClient == nil { - return &statsd.NoOpClient{} + return &statsd.NoOpClientDirect{} } return c.statsdClient } diff --git a/ddtrace/tracer/stats_test.go b/ddtrace/tracer/stats_test.go index 6ca000a600..42f0bc17b0 100644 --- a/ddtrace/tracer/stats_test.go +++ b/ddtrace/tracer/stats_test.go @@ -12,8 +12,10 @@ import ( "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-go/v5/statsd" "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/constants" "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/utils" + "gopkg.in/DataDog/dd-trace-go.v1/internal/statsdtest" ) func TestAlignTs(t *testing.T) { @@ -39,7 +41,7 @@ func TestConcentrator(t *testing.T) { } t.Run("start-stop", func(t *testing.T) { assert := assert.New(t) - c := newConcentrator(&config{}, bucketSize) + c := newConcentrator(&config{}, bucketSize, &statsd.NoOpClientDirect{}) assert.EqualValues(atomic.LoadUint32(&c.stopped), 1) c.Start() assert.EqualValues(atomic.LoadUint32(&c.stopped), 0) @@ -58,7 +60,7 @@ func TestConcentrator(t *testing.T) { t.Run("flusher", func(t *testing.T) { t.Run("old", func(t *testing.T) { transport := newDummyTransport() - c := newConcentrator(&config{transport: transport, env: "someEnv"}, 500_000) + c := newConcentrator(&config{transport: transport, env: "someEnv"}, 500_000, &statsd.NoOpClientDirect{}) assert.Len(t, transport.Stats(), 0) ss1, ok := c.newTracerStatSpan(&s1, nil) assert.True(t, ok) @@ -73,9 +75,10 @@ func TestConcentrator(t *testing.T) { assert.Equal(t, "http.request", actualStats[0].Stats[0].Stats[0].Name) }) - t.Run("recent", func(t *testing.T) { + t.Run("recent+stats", func(t *testing.T) { transport := newDummyTransport() - c := newConcentrator(&config{transport: transport, env: "someEnv"}, (10 * time.Second).Nanoseconds()) + testStats := &statsdtest.TestStatsdClient{} + c := newConcentrator(&config{transport: transport, env: "someEnv"}, (10 * time.Second).Nanoseconds(), testStats) assert.Len(t, transport.Stats(), 0) ss1, ok := c.newTracerStatSpan(&s1, nil) assert.True(t, ok) @@ -96,12 +99,13 @@ func TestConcentrator(t *testing.T) { assert.Len(t, names, 2) assert.NotNil(t, names["http.request"]) assert.NotNil(t, names["potato"]) + assert.Contains(t, testStats.CallNames(), "datadog.tracer.stats.spans_in") }) t.Run("ciGitSha", func(t *testing.T) { utils.AddCITags(constants.GitCommitSHA, "DEADBEEF") transport := newDummyTransport() - c := newConcentrator(&config{transport: transport, env: "someEnv"}, (10 * time.Second).Nanoseconds()) + c := newConcentrator(&config{transport: transport, env: "someEnv"}, (10 * time.Second).Nanoseconds(), &statsd.NoOpClientDirect{}) assert.Len(t, transport.Stats(), 0) ss1, ok := c.newTracerStatSpan(&s1, nil) assert.True(t, ok) @@ -115,7 +119,7 @@ func TestConcentrator(t *testing.T) { // stats should be sent if the concentrator is stopped t.Run("stop", func(t *testing.T) { transport := newDummyTransport() - c := newConcentrator(&config{transport: transport}, 500000) + c := newConcentrator(&config{transport: transport}, 500000, &statsd.NoOpClientDirect{}) assert.Len(t, transport.Stats(), 0) ss1, ok := c.newTracerStatSpan(&s1, nil) assert.True(t, ok) diff --git a/ddtrace/tracer/telemetry.go b/ddtrace/tracer/telemetry.go index bc9185858f..e4dc21fc4d 100644 --- a/ddtrace/tracer/telemetry.go +++ b/ddtrace/tracer/telemetry.go @@ -47,6 +47,7 @@ func startTelemetry(c *config) { {Name: "dogstatsd_port", Value: c.agent.StatsdPort}, {Name: "lambda_mode", Value: c.logToStdout}, {Name: "send_retries", Value: c.sendRetries}, + {Name: "retry_interval", Value: c.retryInterval}, {Name: "trace_startup_logs_enabled", Value: c.logStartup}, {Name: "service", Value: c.serviceName}, {Name: "universal_version", Value: c.universalVersion}, diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 050d397f26..e0c9363de8 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -262,55 +262,90 @@ func (p *chainedPropagator) Inject(spanCtx ddtrace.SpanContext, carrier interfac return nil } -// Extract implements Propagator. This method will attempt to extract the context +// Extract implements Propagator. This method will attempt to extract a span context // based on the precedence order of the propagators. Generally, the first valid -// trace context that could be extracted will be returned, and other extractors will -// be ignored. However, the W3C tracestate header value will always be extracted and -// stored in the local trace context even if a previous propagator has already succeeded -// so long as the trace-ids match. +// trace context that could be extracted will be returned. However, the W3C tracestate +// header value will always be extracted and stored in the local trace context even if +// a previous propagator has succeeded so long as the trace-ids match. +// Furthermore, if we have already successfully extracted a trace context and a +// subsequent trace context has conflicting trace information, such information will +// be relayed in the returned SpanContext with a SpanLink. func (p *chainedPropagator) Extract(carrier interface{}) (ddtrace.SpanContext, error) { var ctx ddtrace.SpanContext + var links []ddtrace.SpanLink for _, v := range p.extractors { - if ctx != nil { - // A local trace context has already been extracted. - pw3c, isW3C := v.(*propagatorW3c) - if !isW3C { - continue // Ignore other propagators. + firstExtract := (ctx == nil) // ctx stores the most recently extracted ctx across iterations; if it's nil, no extractor has run yet + extractedCtx, err := v.Extract(carrier) + if firstExtract { + if err != nil && err != ErrSpanContextNotFound { // We only care if the first extraction returns an error because this breaks distributed tracing + return nil, err } - w3cCtx, err := pw3c.Extract(carrier) - if err == nil && w3cCtx.(*spanContext).TraceID128() == ctx.(*spanContext).TraceID128() { - pw3c.propagateTracestate(ctx.(*spanContext), w3cCtx.(*spanContext)) - if w3cCtx.SpanID() != ctx.SpanID() { - var ddCtx *spanContext - if ddp := getDatadogPropagator(p); ddp != nil { - if ddSpanCtx, err := ddp.Extract(carrier); err == nil { - ddCtx, _ = ddSpanCtx.(*spanContext) + if p.onlyExtractFirst { // Return early if only performing one extraction + return extractedCtx.(*spanContext), nil + } + ctx = extractedCtx + } else { // A local trace context has already been extracted + extractedCtx2, ok1 := extractedCtx.(*spanContext) + ctx2, ok2 := ctx.(*spanContext) + // If we can't cast to spanContext, we can't propgate tracestate or create span links + if !ok1 || !ok2 { + continue + } + if extractedCtx2.TraceID128() == ctx2.TraceID128() { + if pW3C, ok := v.(*propagatorW3c); ok { + pW3C.propagateTracestate(ctx2, extractedCtx2) + // If trace IDs match but span IDs do not, use spanID from `*propagatorW3c` extractedCtx for parenting + if extractedCtx2.SpanID() != ctx2.SpanID() { + var ddCtx *spanContext + // Grab the datadog-propagated spancontext again + if ddp := getDatadogPropagator(p); ddp != nil { + if ddSpanCtx, err := ddp.Extract(carrier); err == nil { + ddCtx, _ = ddSpanCtx.(*spanContext) + } } + overrideDatadogParentID(ctx2, extractedCtx2, ddCtx) } - overrideDatadogParentID(ctx.(*spanContext), w3cCtx.(*spanContext), ddCtx) } + } else { // Trace IDs do not match - create span links + link := ddtrace.SpanLink{TraceID: extractedCtx2.TraceID(), SpanID: extractedCtx2.SpanID(), TraceIDHigh: extractedCtx2.TraceIDUpper(), Attributes: map[string]string{"reason": "terminated_context", "context_headers": getPropagatorName(v)}} + if trace := extractedCtx2.trace; trace != nil { + if flags := uint32(*trace.priority); flags > 0 { // Set the flags based on the sampling priority + link.Flags = 1 + } else { + link.Flags = 0 + } + link.Tracestate = extractedCtx2.trace.propagatingTag(tracestateHeader) + } + links = append(links, link) } - break - } - var err error - ctx, err = v.Extract(carrier) - if ctx != nil { - if p.onlyExtractFirst { - // Return early if the customer configured that only the first successful - // extraction should occur. - return ctx, nil - } - } else if err != ErrSpanContextNotFound { - return nil, err } } + // 0 successful extractions if ctx == nil { return nil, ErrSpanContextNotFound } + if spCtx, ok := ctx.(*spanContext); ok && len(links) > 0 { + spCtx.spanLinks = links + } log.Debugf("Extracted span context: %#v", ctx) return ctx, nil } +func getPropagatorName(p Propagator) string { + switch p.(type) { + case *propagator: + return "datadog" + case *propagatorB3: + return "b3multi" + case *propagatorB3SingleHeader: + return "b3" + case *propagatorW3c: + return "tracecontext" + default: + return "" + } +} + // propagateTracestate will add the tracestate propagating tag to the given // *spanContext. The W3C trace context will be extracted from the provided // carrier. The trace id of this W3C trace context must match the trace id @@ -503,9 +538,13 @@ func getDatadogPropagator(cp *chainedPropagator) *propagator { return nil } -// overrideDatadogParentID overrides the span ID of a context with the ID extracted from tracecontext headers -// if the reparenting ID is not set on the context, the span ID from datadog headers is used. +// overrideDatadogParentID overrides the span ID of a context with the ID extracted from tracecontext headers. +// If the reparenting ID is not set on the context, the span ID from datadog headers is used. +// spanContexts are passed by reference to avoid copying lock value in spanContext type func overrideDatadogParentID(ctx, w3cCtx, ddCtx *spanContext) { + if ctx == nil || w3cCtx == nil || ddCtx == nil { + return + } ctx.spanID = w3cCtx.spanID if w3cCtx.reparentID != "" { ctx.reparentID = w3cCtx.reparentID diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index bbbd3b52c9..74038fc0c7 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -209,7 +209,9 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { ctx, err := tracer.Extract(headers) assert.Nil(err) sctx, ok := ctx.(*spanContext) - assert.True(ok) + if !ok { + t.Fail() + } assert.Equal("00000000000000000000000000000004", sctx.traceID.HexEncoded()) if tc.conflictingParentID == true { // tracecontext span id should be used @@ -669,7 +671,7 @@ func TestEnvVars(t *testing.T) { } for _, test := range tests { t.Run(fmt.Sprintf("inject with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() root := tracer.StartSpan("web.request").(*span) ctx, ok := root.Context().(*spanContext) @@ -741,7 +743,7 @@ func TestEnvVars(t *testing.T) { } for _, test := range tests { t.Run(fmt.Sprintf("extract with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(test.in) @@ -780,7 +782,7 @@ func TestEnvVars(t *testing.T) { } for _, tc := range tests { t.Run(fmt.Sprintf("extract with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) _, err := tracer.Extract(tc.in) @@ -837,7 +839,7 @@ func TestEnvVars(t *testing.T) { } for _, tc := range tests { t.Run(fmt.Sprintf("extract with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc.in) @@ -874,7 +876,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("b3 single header inject #%d", i), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() root := tracer.StartSpan("myrequest").(*span) ctx, ok := root.Context().(*spanContext) @@ -932,7 +934,7 @@ func TestEnvVars(t *testing.T) { } for _, tc := range tests { t.Run(fmt.Sprintf("inject with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithPropagator(NewPropagator(&PropagatorConfig{B3: true})), WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithPropagator(NewPropagator(&PropagatorConfig{B3: true})), WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() root := tracer.StartSpan("web.request").(*span) ctx, ok := root.Context().(*spanContext) @@ -1007,7 +1009,7 @@ func TestEnvVars(t *testing.T) { } for _, tc := range tests { t.Run(fmt.Sprintf("extract with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) @@ -1067,7 +1069,7 @@ func TestEnvVars(t *testing.T) { } for _, tc := range tests { t.Run(fmt.Sprintf("inject and extract with env=%q", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() root := tracer.StartSpan("web.request").(*span) root.SetTag(ext.SamplingPriority, -1) @@ -1279,7 +1281,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%v extract/valid with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc.in) @@ -1335,7 +1337,7 @@ func TestEnvVars(t *testing.T) { for i, tc := range tests { t.Run(fmt.Sprintf("#%v extract/invalid with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc) @@ -1387,7 +1389,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%v extract/valid with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc.inHeaders) @@ -1589,7 +1591,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%d w3c inject with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) root := tracer.StartSpan("web.request").(*span) @@ -1619,7 +1621,7 @@ func TestEnvVars(t *testing.T) { }) t.Run(fmt.Sprintf("w3c inject with env=%q / testing tag list-member limit", testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) root := tracer.StartSpan("web.request").(*span) @@ -1687,7 +1689,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc.inHeaders) @@ -1762,7 +1764,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%d w3c inject/extract with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc.in) @@ -1826,7 +1828,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%d w3c inject/extract with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) pCtx, err := tracer.Extract(tc.in) @@ -1909,7 +1911,7 @@ func TestEnvVars(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("#%v extract with env=%q", i, testEnv), func(t *testing.T) { - tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClient{})) + tracer := newTracer(WithHTTPClient(c), withStatsdClient(&statsd.NoOpClientDirect{})) defer tracer.Stop() assert := assert.New(t) ctx, err := tracer.Extract(tc.in) @@ -1957,6 +1959,95 @@ func TestTraceContextPrecedence(t *testing.T) { assert.Equal(2, p) } +// Assert that span links are generated only when trace headers contain divergent trace IDs +func TestSpanLinks(t *testing.T) { + s, c := httpmem.ServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + })) + defer s.Close() + t.Run("Links on divergent trace IDs", func(t *testing.T) { + carrier := TextMapCarrier{ + DefaultTraceIDHeader: "1", + DefaultParentIDHeader: "1", + DefaultPriorityHeader: "3", + traceparentHeader: "00-00000000000000000000000000000002-0000000000000002-01", + tracestateHeader: "dd=s:1;o:rum;t.usr.id:baz64~~", + b3TraceIDHeader: "3", + b3SpanIDHeader: "3", + } + w3cLink := ddtrace.SpanLink{TraceID: 2, TraceIDHigh: 0, SpanID: 2, Tracestate: "dd=s:1;o:rum;t.usr.id:baz64~~", Flags: 1, Attributes: map[string]string{"reason": "terminated_context", "context_headers": "tracecontext"}} + ddLink := ddtrace.SpanLink{TraceID: 1, TraceIDHigh: 0, SpanID: 1, Flags: 1, Attributes: map[string]string{"reason": "terminated_context", "context_headers": "datadog"}} + b3Link := ddtrace.SpanLink{TraceID: 3, TraceIDHigh: 0, SpanID: 3, Tracestate: "", Flags: 0, Attributes: map[string]string{"reason": "terminated_context", "context_headers": "b3multi"}} + tests := []struct { + name string + envVal string + out []ddtrace.SpanLink + tid traceID + }{ + { + name: "datadog first", + envVal: "datadog,tracecontext,b3", + out: []ddtrace.SpanLink{w3cLink, b3Link}, + tid: traceIDFrom64Bits(1), + }, + { + name: "tracecontext first", + envVal: "tracecontext,datadog,b3", + out: []ddtrace.SpanLink{ddLink, b3Link}, + tid: traceIDFrom64Bits(2), + }, + { + name: "b3 first", + envVal: "b3,tracecontext,datadog", + out: []ddtrace.SpanLink{w3cLink, ddLink}, + tid: traceIDFrom64Bits(3), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envVal != "" { + t.Setenv(headerPropagationStyleExtract, tt.envVal) + } + tracer := newTracer(WithHTTPClient(c)) + defer tracer.Stop() + assert := assert.New(t) + ctx, err := tracer.Extract(carrier) + if err != nil { + t.Fatal(err) + } + sctx, ok := ctx.(*spanContext) + assert.True(ok) + + assert.Equal(tt.tid, sctx.traceID) + assert.Len(sctx.spanLinks, 2) + assert.Contains(sctx.spanLinks, tt.out[0]) + assert.Contains(sctx.spanLinks, tt.out[1]) + }) + } + }) + t.Run("No links on equal trace IDs", func(t *testing.T) { + carrier := TextMapCarrier{ + DefaultTraceIDHeader: "1", + DefaultParentIDHeader: "1", + DefaultPriorityHeader: "3", + traceparentHeader: "00-00000000000000000000000000000001-0000000000000002-01", + tracestateHeader: "dd=s:1;o:rum;t.usr.id:baz64~~", + } + tracer := newTracer(WithHTTPClient(c)) + defer tracer.Stop() + assert := assert.New(t) + ctx, err := tracer.Extract(carrier) + if err != nil { + t.Fatal(err) + } + sctx, ok := ctx.(*spanContext) + assert.True(ok) + + assert.Equal(traceIDFrom64Bits(1), sctx.traceID) + assert.Len(sctx.spanLinks, 0) + }) +} + func TestW3CExtractsBaggage(t *testing.T) { tracer := newTracer() defer tracer.Stop() diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 8e586fe31f..794885095a 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -165,6 +165,19 @@ func Start(opts ...StartOption) { if t.dataStreams != nil { t.dataStreams.Start() } + if t.config.ciVisibilityAgentless { + // CI Visibility agentless mode doesn't require remote configuration. + + // start instrumentation telemetry unless it is disabled through the + // DD_INSTRUMENTATION_TELEMETRY_ENABLED env var + startTelemetry(t.config) + + // start appsec + appsec.Start(t.config.appsecStartOptions...) + _ = t.hostname() // Prime the hostname cache + return + } + // Start AppSec with remote configuration cfg := remoteconfig.DefaultClientConfig() cfg.AgentURL = t.config.agentURL.String() @@ -300,14 +313,13 @@ func newUnstartedTracer(opts ...StartOption) *tracer { prioritySampling: sampler, pid: os.Getpid(), logDroppedTraces: time.NewTicker(1 * time.Second), - stats: newConcentrator(c, defaultStatsBucketSize), + stats: newConcentrator(c, defaultStatsBucketSize, statsd), obfuscator: obfuscate.NewObfuscator(obfuscate.Config{ SQL: obfuscate.SQLConfig{ TableNames: c.agent.HasFlag("table_names"), ReplaceDigits: c.agent.HasFlag("quantize_sql_tables") || c.agent.HasFlag("replace_sql_digits"), KeepSQLAlias: c.agent.HasFlag("keep_sql_alias"), DollarQuotedFunc: c.agent.HasFlag("dollar_quoted_func"), - Cache: c.agent.HasFlag("sql_cache"), }, }), statsd: statsd, diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 038107843e..dd1820621e 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -25,6 +25,7 @@ import ( "time" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" + "github.com/DataDog/datadog-go/v5/statsd" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal" @@ -2346,7 +2347,7 @@ func TestFlush(t *testing.T) { tr.statsd = ts transport := newDummyTransport() - c := newConcentrator(&config{transport: transport, env: "someEnv"}, defaultStatsBucketSize) + c := newConcentrator(&config{transport: transport, env: "someEnv"}, defaultStatsBucketSize, &statsd.NoOpClientDirect{}) tr.stats = c c.Start() defer c.Stop() diff --git a/ddtrace/tracer/writer.go b/ddtrace/tracer/writer.go index ca663c07b7..e83b31d6dc 100644 --- a/ddtrace/tracer/writer.go +++ b/ddtrace/tracer/writer.go @@ -105,7 +105,7 @@ func (h *agentTraceWriter) flush() { var err error for attempt := 0; attempt <= h.config.sendRetries; attempt++ { size, count = p.size(), p.itemCount() - log.Debugf("Sending payload: size: %d traces: %d\n", size, count) + log.Debugf("Attempt to send payload: size: %d traces: %d\n", size, count) var rc io.ReadCloser rc, err = h.config.transport.send(p) if err == nil { @@ -119,7 +119,7 @@ func (h *agentTraceWriter) flush() { } log.Errorf("failure sending traces (attempt %d), will retry: %v", attempt+1, err) p.reset() - time.Sleep(time.Millisecond) + time.Sleep(h.config.retryInterval) } h.statsd.Count("datadog.tracer.traces_dropped", int64(count), []string{"reason:send_failed"}, 1) log.Errorf("lost %d traces: %v", count, err) diff --git a/ddtrace/tracer/writer_test.go b/ddtrace/tracer/writer_test.go index a7717c9917..bcdf529a58 100644 --- a/ddtrace/tracer/writer_test.go +++ b/ddtrace/tracer/writer_test.go @@ -14,6 +14,7 @@ import ( "math" "strings" "testing" + "time" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "gopkg.in/DataDog/dd-trace-go.v1/internal/statsdtest" @@ -363,26 +364,30 @@ func (t *failingTransport) send(p *payload) (io.ReadCloser, error) { func TestTraceWriterFlushRetries(t *testing.T) { testcases := []struct { configRetries int + retryInterval time.Duration failCount int tracesSent bool expAttempts int }{ - {configRetries: 0, failCount: 0, tracesSent: true, expAttempts: 1}, - {configRetries: 0, failCount: 1, tracesSent: false, expAttempts: 1}, + {configRetries: 0, retryInterval: time.Millisecond, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 0, retryInterval: time.Millisecond, failCount: 1, tracesSent: false, expAttempts: 1}, - {configRetries: 1, failCount: 0, tracesSent: true, expAttempts: 1}, - {configRetries: 1, failCount: 1, tracesSent: true, expAttempts: 2}, - {configRetries: 1, failCount: 2, tracesSent: false, expAttempts: 2}, + {configRetries: 1, retryInterval: time.Millisecond, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 1, retryInterval: time.Millisecond, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 1, retryInterval: time.Millisecond, failCount: 2, tracesSent: false, expAttempts: 2}, - {configRetries: 2, failCount: 0, tracesSent: true, expAttempts: 1}, - {configRetries: 2, failCount: 1, tracesSent: true, expAttempts: 2}, - {configRetries: 2, failCount: 2, tracesSent: true, expAttempts: 3}, - {configRetries: 2, failCount: 3, tracesSent: false, expAttempts: 3}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 2, tracesSent: true, expAttempts: 3}, + {configRetries: 2, retryInterval: time.Millisecond, failCount: 3, tracesSent: false, expAttempts: 3}, + + {configRetries: 1, retryInterval: 2 * time.Millisecond, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 2, retryInterval: 2 * time.Millisecond, failCount: 2, tracesSent: true, expAttempts: 3}, } sentCounts := map[string]int64{ "datadog.tracer.decode_error": 1, - "datadog.tracer.flush_bytes": 197, + "datadog.tracer.flush_bytes": 185, "datadog.tracer.flush_traces": 1, } droppedCounts := map[string]int64{ @@ -401,14 +406,16 @@ func TestTraceWriterFlushRetries(t *testing.T) { c := newConfig(func(c *config) { c.transport = p c.sendRetries = test.configRetries + c.retryInterval = test.retryInterval }) var statsd statsdtest.TestStatsdClient h := newAgentTraceWriter(c, nil, &statsd) h.add(ss) - + start := time.Now() h.flush() h.wg.Wait() + elapsed := time.Since(start) assert.Equal(test.expAttempts, p.sendAttempts) assert.Equal(test.tracesSent, p.tracesSent) @@ -419,10 +426,20 @@ func TestTraceWriterFlushRetries(t *testing.T) { } else { assert.Equal(droppedCounts, statsd.Counts()) } + if test.configRetries > 0 && test.failCount > 1 { + assert.GreaterOrEqual(elapsed, test.retryInterval*time.Duration(minInts(test.configRetries+1, test.failCount))) + } }) } } +func minInts(a, b int) int { + if a < b { + return a + } + return b +} + func BenchmarkJsonEncodeSpan(b *testing.B) { s := makeSpan(10) s.Metrics["nan"] = math.NaN() diff --git a/go.mod b/go.mod index 16b5340001..62452c745f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/DataDog/datadog-agent/pkg/trace v0.58.0 github.com/DataDog/datadog-go/v5 v5.5.0 github.com/DataDog/go-libddwaf/v3 v3.5.1 - github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59 + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 github.com/DataDog/gostackparse v0.7.0 github.com/DataDog/sketches-go v1.4.5 github.com/IBM/sarama v1.40.0 @@ -40,6 +40,7 @@ require ( github.com/elastic/go-elasticsearch/v8 v8.4.0 github.com/emicklei/go-restful v2.16.0+incompatible github.com/emicklei/go-restful/v3 v3.11.0 + github.com/envoyproxy/go-control-plane v0.12.0 github.com/garyburd/redigo v1.6.4 github.com/gin-gonic/gin v1.9.1 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 @@ -154,6 +155,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -162,6 +164,7 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/ebitengine/purego v0.6.0-alpha.5 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect diff --git a/go.sum b/go.sum index e0ef88aefa..8388b569ef 100644 --- a/go.sum +++ b/go.sum @@ -644,8 +644,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59 h1:s4hgS6gqbXIakEMMujYiHCVVsB3R3oZtqEzPBMnFU2w= -github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 h1:bpitH5JbjBhfcTG+H2RkkiUXpYa8xSuIPnyNtTaSPog= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= @@ -890,6 +890,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 h1:DBmgJDC9dTfkVyGgipamEh2BpGYxScCH1TOF1LL1cXc= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= @@ -1118,9 +1120,13 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= diff --git a/integration_go.mod b/integration_go.mod new file mode 100644 index 0000000000..0bf48eaca0 --- /dev/null +++ b/integration_go.mod @@ -0,0 +1,91 @@ +module gopkg.in/DataDog/dd-trace-go.v1 // This file is manually maintained and updated + +go 1.22.0 + +require ( + cloud.google.com/go/pubsub v1.33.0 + github.com/99designs/gqlgen v0.17.36 + github.com/IBM/sarama v1.40.0 + github.com/Shopify/sarama v1.38.1 + github.com/aws/aws-sdk-go v1.44.327 + github.com/aws/aws-sdk-go-v2 v1.20.3 + github.com/aws/aws-sdk-go-v2/config v1.18.21 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.21.4 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.93.2 + github.com/aws/aws-sdk-go-v2/service/eventbridge v1.20.4 + github.com/aws/aws-sdk-go-v2/service/kinesis v1.18.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.32.0 + github.com/aws/aws-sdk-go-v2/service/sfn v1.19.4 + github.com/aws/aws-sdk-go-v2/service/sns v1.21.4 + github.com/aws/aws-sdk-go-v2/service/sqs v1.24.4 + github.com/bradfitz/gomemcache v0.0.0-20230611145640-acc696258285 + github.com/confluentinc/confluent-kafka-go v1.9.2 + github.com/confluentinc/confluent-kafka-go/v2 v2.2.0 + github.com/denisenkom/go-mssqldb v0.11.0 + github.com/dimfeld/httptreemux/v5 v5.5.0 + github.com/elastic/go-elasticsearch/v6 v6.8.5 + github.com/elastic/go-elasticsearch/v7 v7.17.10 + github.com/elastic/go-elasticsearch/v8 v8.15.0 + github.com/emicklei/go-restful v2.16.0+incompatible + github.com/emicklei/go-restful/v3 v3.11.0 + github.com/garyburd/redigo v1.6.4 + github.com/gin-gonic/gin v1.9.1 + github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 + github.com/go-chi/chi v1.5.4 + github.com/go-chi/chi/v5 v5.0.10 + github.com/go-pg/pg/v10 v10.11.1 + github.com/go-redis/redis v6.15.9+incompatible + github.com/go-redis/redis/v7 v7.4.1 + github.com/go-redis/redis/v8 v8.11.5 + github.com/go-redis/redis/v9 v9.7.0 // renamed to redis/go-redis in v9 + github.com/go-sql-driver/mysql v1.6.0 + github.com/gocql/gocql v1.6.0 + github.com/gofiber/fiber/v2 v2.52.5 + github.com/gomodule/redigo v1.8.9 + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b + github.com/google/uuid v1.5.0 + github.com/gorilla/mux v1.8.0 + github.com/graph-gophers/graphql-go v1.5.0 + github.com/graphql-go/graphql v0.8.1 + github.com/graphql-go/handler v0.2.3 + github.com/hashicorp/consul/api v1.24.0 + github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 + github.com/hashicorp/vault/api v1.9.2 + github.com/hashicorp/vault/sdk v0.9.2 + github.com/jackc/pgx/v5 v5.6.0 + github.com/jinzhu/gorm v1.9.16 + github.com/jmoiron/sqlx v1.3.5 + github.com/julienschmidt/httprouter v1.3.0 + github.com/labstack/echo v3.3.10+incompatible + github.com/labstack/echo/v4 v4.11.1 + github.com/lib/pq v1.10.2 + github.com/mattn/go-sqlite3 v1.14.18 + github.com/microsoft/go-mssqldb v0.21.0 + github.com/miekg/dns v1.1.55 + github.com/redis/go-redis/v9 v9.7.0 + github.com/segmentio/kafka-go v0.4.42 + github.com/sirupsen/logrus v1.9.3 + github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d + github.com/tidwall/buntdb v1.3.0 + github.com/twitchtv/twirp v8.1.3+incompatible + github.com/uptrace/bun v1.1.17 + github.com/uptrace/bun/dialect/sqlitedialect v1.1.17 + github.com/urfave/negroni v1.0.0 + github.com/valyala/fasthttp v1.51.0 + github.com/vektah/gqlparser/v2 v2.5.16 + github.com/zenazn/goji v1.0.1 + go.mongodb.org/mongo-driver v1.12.1 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 + google.golang.org/api v0.128.0 + google.golang.org/grpc v1.57.1 + google.golang.org/protobuf v1.33.0 + gopkg.in/jinzhu/gorm.v1 v1.9.2 + gopkg.in/olivere/elastic.v3 v3.0.75 + gopkg.in/olivere/elastic.v5 v5.0.84 + gorm.io/driver/mysql v1.0.1 + gorm.io/driver/postgres v1.4.6 + gorm.io/driver/sqlserver v1.4.2 + gorm.io/gorm v1.25.3 + k8s.io/client-go v0.23.17 +) \ No newline at end of file diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 7be8bd55a2..d90e69002e 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -14,7 +14,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.58.0 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.58.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect - github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59 // indirect + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 // indirect github.com/DataDog/go-sqllexer v0.0.14 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect diff --git a/internal/apps/go.sum b/internal/apps/go.sum index 8723a714a3..1ca195d002 100644 --- a/internal/apps/go.sum +++ b/internal/apps/go.sum @@ -16,8 +16,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59 h1:s4hgS6gqbXIakEMMujYiHCVVsB3R3oZtqEzPBMnFU2w= -github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 h1:bpitH5JbjBhfcTG+H2RkkiUXpYa8xSuIPnyNtTaSPog= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= diff --git a/internal/appsec/testdata/user_rules.json b/internal/appsec/testdata/user_rules.json index 6acb14089e..a13a0d67ed 100644 --- a/internal/appsec/testdata/user_rules.json +++ b/internal/appsec/testdata/user_rules.json @@ -53,6 +53,33 @@ "block" ] }, + { + "id": "tst-037-008", + "name": "Test block on cookies", + "tags": { + "type": "lfi", + "crs_id": "000008", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + } + ], + "regex": "jdfoSDGFkivRG_234" + }, + "operator": "match_regex" + } + ], + "transformers": [], + "on_match": [ + "block" + ] + }, + { "id": "headers-003", "name": "query match", diff --git a/internal/civisibility/utils/net/settings_api.go b/internal/civisibility/utils/net/settings_api.go index 49b190dae4..c6ced4c6d2 100644 --- a/internal/civisibility/utils/net/settings_api.go +++ b/internal/civisibility/utils/net/settings_api.go @@ -120,6 +120,9 @@ func (c *client) GetSettings() (*SettingsResponseData, error) { if responseObject.Data.Attributes.EarlyFlakeDetection.Enabled { settingsResponseType = append(settingsResponseType, telemetry.EfdEnabledSettingsResponseType...) } + if responseObject.Data.Attributes.FlakyTestRetriesEnabled { + settingsResponseType = append(settingsResponseType, telemetry.FlakyTestRetriesEnabledSettingsResponseType...) + } telemetry.GitRequestsSettingsResponse(settingsResponseType) return &responseObject.Data.Attributes, nil } diff --git a/internal/civisibility/utils/telemetry/telemetry.go b/internal/civisibility/utils/telemetry/telemetry.go index 3b700150a7..69d0b039c9 100644 --- a/internal/civisibility/utils/telemetry/telemetry.go +++ b/internal/civisibility/utils/telemetry/telemetry.go @@ -113,9 +113,10 @@ const ( type SettingsResponseType []string var ( - CoverageEnabledSettingsResponseType SettingsResponseType = []string{"coverage_enabled"} - ItrSkipEnabledSettingsResponseType SettingsResponseType = []string{"itrskip_enabled"} - EfdEnabledSettingsResponseType SettingsResponseType = []string{"early_flake_detection_enabled:true"} + CoverageEnabledSettingsResponseType SettingsResponseType = []string{"coverage_enabled"} + ItrSkipEnabledSettingsResponseType SettingsResponseType = []string{"itrskip_enabled"} + EfdEnabledSettingsResponseType SettingsResponseType = []string{"early_flake_detection_enabled:true"} + FlakyTestRetriesEnabledSettingsResponseType SettingsResponseType = []string{"flaky_test_retries_enabled:true"} ) // removeEmptyStrings removes empty string values inside an array or use the same if not empty string is found. diff --git a/internal/datastreams/processor_test.go b/internal/datastreams/processor_test.go index 1e0418a2d8..d60a01c4ef 100644 --- a/internal/datastreams/processor_test.go +++ b/internal/datastreams/processor_test.go @@ -264,7 +264,7 @@ func BenchmarkSetCheckpoint(b *testing.B) { client := &http.Client{ Transport: &noOpTransport{}, } - p := NewProcessor(&statsd.NoOpClient{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client) + p := NewProcessor(&statsd.NoOpClientDirect{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client) p.Start() for i := 0; i < b.N; i++ { p.SetCheckpointWithParams(context.Background(), options.CheckpointParams{PayloadSize: 1000}, "type:edge-1", "direction:in", "type:kafka", "topic:topic1", "group:group1") diff --git a/internal/exectracetest/go.mod b/internal/exectracetest/go.mod index 09917ff4f6..33a74a94ba 100644 --- a/internal/exectracetest/go.mod +++ b/internal/exectracetest/go.mod @@ -19,7 +19,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.58.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.5.1 // indirect - github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59 // indirect + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 // indirect github.com/DataDog/go-sqllexer v0.0.14 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect diff --git a/internal/exectracetest/go.sum b/internal/exectracetest/go.sum index 9e64886e7e..7321d5519b 100644 --- a/internal/exectracetest/go.sum +++ b/internal/exectracetest/go.sum @@ -16,8 +16,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.5.1 h1:GWA4ln4DlLxiXm+X7HA/oj0ZLcdCwOS81KQitegRTyY= github.com/DataDog/go-libddwaf/v3 v3.5.1/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59 h1:s4hgS6gqbXIakEMMujYiHCVVsB3R3oZtqEzPBMnFU2w= -github.com/DataDog/go-runtime-metrics-internal v0.0.0-20241106155157-194426bbbd59/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 h1:bpitH5JbjBhfcTG+H2RkkiUXpYa8xSuIPnyNtTaSPog= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= github.com/DataDog/go-sqllexer v0.0.14 h1:xUQh2tLr/95LGxDzLmttLgTo/1gzFeOyuwrQa/Iig4Q= github.com/DataDog/go-sqllexer v0.0.14/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= diff --git a/internal/statsd.go b/internal/statsd.go index df1d18a307..60f3d4431c 100644 --- a/internal/statsd.go +++ b/internal/statsd.go @@ -19,6 +19,7 @@ type StatsdClient interface { CountWithTimestamp(name string, value int64, tags []string, rate float64, timestamp time.Time) error Gauge(name string, value float64, tags []string, rate float64) error GaugeWithTimestamp(name string, value float64, tags []string, rate float64, timestamp time.Time) error + DistributionSamples(name string, values []float64, tags []string, rate float64) error Timing(name string, value time.Duration, tags []string, rate float64) error Flush() error Close() error @@ -29,9 +30,9 @@ func NewStatsdClient(addr string, globalTags []string) (StatsdClient, error) { if addr == "" { addr = DefaultDogstatsdAddr } - client, err := statsd.New(addr, statsd.WithMaxMessagesPerPayload(40), statsd.WithTags(globalTags)) + client, err := statsd.NewDirect(addr, statsd.WithMaxMessagesPerPayload(40), statsd.WithTags(globalTags)) if err != nil { - return &statsd.NoOpClient{}, err + return &statsd.NoOpClientDirect{}, err } return client, nil } diff --git a/internal/statsdtest/statsdtest.go b/internal/statsdtest/statsdtest.go index 8845e6465c..ffcd700cd1 100644 --- a/internal/statsdtest/statsdtest.go +++ b/internal/statsdtest/statsdtest.go @@ -11,6 +11,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "gopkg.in/DataDog/dd-trace-go.v1/internal" ) type callType int64 @@ -24,6 +25,8 @@ const ( callTypeTiming ) +var _ internal.StatsdClient = &TestStatsdClient{} + type TestStatsdClient struct { mu sync.RWMutex gaugeCalls []TestStatsdCall @@ -104,6 +107,10 @@ func (tg *TestStatsdClient) CountWithTimestamp(name string, value int64, tags [] }) } +func (tg *TestStatsdClient) DistributionSamples(_ string, _ []float64, _ []string, _ float64) error { + panic("not implemented") +} + func (tg *TestStatsdClient) Timing(name string, value time.Duration, tags []string, rate float64) error { return tg.addMetric(callTypeTiming, tags, TestStatsdCall{ name: name, diff --git a/internal/telemetry/utils.go b/internal/telemetry/utils.go index 96c212b176..d2e454ecc1 100644 --- a/internal/telemetry/utils.go +++ b/internal/telemetry/utils.go @@ -12,7 +12,6 @@ import ( "math" "sort" "strings" - "testing" ) // MockGlobalClient replaces the global telemetry client with a custom @@ -30,8 +29,16 @@ func MockGlobalClient(client Client) func() { } } +// Errorfer is an interface that allows to call the `Errorf` method. +// This is the same interface as `testing.T` because the goal of this +// interface is to remove the need to import `testing` in this package +// that is actually imported by users. +type Errorfer interface { + Errorf(format string, args ...any) +} + // Check is a testing utility to assert that a target key in config contains the expected value -func Check(t *testing.T, configuration []Configuration, key string, expected interface{}) { +func Check(t Errorfer, configuration []Configuration, key string, expected interface{}) { for _, kv := range configuration { if kv.Name == key { if kv.Value != expected { diff --git a/internal/version/version.go b/internal/version/version.go index 6918f58dce..c91fed8dcc 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ import ( // Tag specifies the current release tag. It needs to be manually // updated. A test checks that the value of Tag never points to a // git tag that is older than HEAD. -const Tag = "v1.71.0-dev" +const Tag = "v1.72.0-dev" // Dissected version number. Filled during init() var (