feat: add process tags to traces

Matrixfile

@@ -14,6 +14,9 @@
   'core_with_libdatadog_api' => {
     '' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby',
   },
+  'core_with_rails' => {
+    'rails8' => '❌ 2.5 / ❌ 2.6 / ❌ 2.7 / ❌ 3.0 / ❌ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby',
+  },
   'error_tracking' => {
     '' => '❌ 2.5 / ❌ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ 3.5 / ❌ jruby',
   },

Rakefile

@@ -85,7 +85,7 @@ namespace :spec do
     :graphql, :graphql_unified_trace_patcher, :graphql_trace_patcher, :graphql_tracing_patcher,
     :rails, :railsredis, :railsredis_activesupport, :railsactivejob,
     :elasticsearch, :http, :redis, :sidekiq, :sinatra, :hanami, :hanami_autoinstrument,
-    :profiling, :core_with_libdatadog_api, :error_tracking, :open_feature]
+    :profiling, :core_with_libdatadog_api, :error_tracking, :open_feature, :core_with_rails]
 
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:main) do |t, args|
@@ -232,6 +232,12 @@ namespace :spec do
   end
   # rubocop:enable Style/MultilineBlockChain
 
+  desc '' # "Explicitly hiding from `rake -T`"
+  RSpec::Core::RakeTask.new(:core_with_rails) do |t, args|
+    t.pattern = 'spec/datadog/core/environment/process_spec.rb'
+    t.rspec_opts = args.to_a.join(' ')
+  end
+
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:error_tracking) do |t, args|
     t.pattern = 'spec/datadog/error_tracking/**/*_spec.rb'

lib/datadog/core/configuration/settings.rb

@@ -1003,6 +1003,16 @@ def initialize(*_)
           end
         end
 
+        # Enable experimental process tags propagation such that payloads like spans contain the process tag.
+        #
+        # @default `DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED` environment variable, otherwise `false`
+        # @return [Boolean]
+        option :experimental_propagate_process_tags_enabled do |o|
+          o.env 'DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED'
+          o.default false
+          o.type :bool
+        end
+
         # Tracer specific configuration starting with APM (e.g. DD_APM_TRACING_ENABLED).
         # @public_api
         settings :apm do

lib/datadog/core/configuration/supported_configurations.rb

@@ -44,6 +44,7 @@ module Configuration
          "DD_ERROR_TRACKING_HANDLED_ERRORS" => {version: ["A"]},
          "DD_ERROR_TRACKING_HANDLED_ERRORS_INCLUDE" => {version: ["A"]},
          "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED" => {version: ["A"]},
+         "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED" => {version: ["A"]},
          "DD_GIT_COMMIT_SHA" => {version: ["A"]},
          "DD_GIT_REPOSITORY_URL" => {version: ["A"]},
          "DD_HEALTH_METRICS_ENABLED" => {version: ["A"]},

lib/datadog/core/environment/ext.rb

@@ -33,8 +33,14 @@ module Ext
         LANG_INTERPRETER = "#{RUBY_ENGINE}-#{RUBY_PLATFORM}"
         LANG_PLATFORM = RUBY_PLATFORM
         LANG_VERSION = RUBY_VERSION
+        PROCESS_TYPE = 'script' # Out of the options [jar, script, class, executable], we consider Ruby to always be a script
         RUBY_ENGINE = ::RUBY_ENGINE # e.g. 'ruby', 'jruby', 'truffleruby'
         TAG_ENV = 'env'
+        TAG_ENTRYPOINT_BASEDIR = "entrypoint.basedir"
+        TAG_ENTRYPOINT_NAME = "entrypoint.name"
+        TAG_ENTRYPOINT_WORKDIR = "entrypoint.workdir"
+        TAG_ENTRYPOINT_TYPE = "entrypoint.type"
+        TAG_PROCESS_TAGS = "_dd.tags.process"
         TAG_SERVICE = 'service'
         TAG_VERSION = 'version'
 

lib/datadog/core/environment/process.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+require_relative '../normalizer'
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves process level information such that it can be attached to various payloads
+      module Process
+        extend self
+
+        # This method returns a key/value part of serialized tags in the format of k1:v1,k2:v2,k3:v3
+        # @return [String] comma-separated normalized key:value pairs
+        def serialized
+          return @serialized if defined?(@serialized)
+          tags = []
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_WORKDIR}:#{Normalizer.normalize(entrypoint_workdir, remove_digit_start_char: false)}" if entrypoint_workdir
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_NAME}:#{Normalizer.normalize(entrypoint_name, remove_digit_start_char: false)}" if entrypoint_name
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_BASEDIR}:#{Normalizer.normalize(entrypoint_basedir, remove_digit_start_char: false)}" if entrypoint_basedir
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_TYPE}:#{Normalizer.normalize(entrypoint_type, remove_digit_start_char: false)}" if entrypoint_type
+          @serialized = tags.join(',').freeze
+        end
+
+        private
+
+        # Returns the last segment of the working directory of the process
+        # Example: /app/myapp -> myapp
+        # @return [String] the last segment of the working directory
+        def entrypoint_workdir
+          File.basename(Dir.pwd)
+        end
+
+        # Returns the entrypoint type of the process
+        # In Ruby, the entrypoint type is always 'script'
+        # @return [String] the type of the process, which is fixed in Ruby
+        def entrypoint_type
+          Environment::Ext::PROCESS_TYPE
+        end
+
+        # Returns the last segment of the base directory of the process
+        # Example 1: /bin/mybin -> mybin
+        # Example 2: ruby /test/myapp.rb -> myapp
+        # @return [String] the last segment of base directory of the script
+        def entrypoint_name
+          File.basename($0)
+        end
+
+        # Returns the last segment of the base directory of the process
+        # Example 1: /bin/mybin -> bin
+        # Example 2: ruby /test/myapp.js -> test
+        # @return [String] the last segment of the base directory of the script
+        def entrypoint_basedir
+          File.basename(File.expand_path(File.dirname($0)))
+        end
+      end
+    end
+  end
+end

lib/datadog/core/normalizer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Normalizer
+      module_function
+
+      INVALID_TAG_CHARACTERS = %r{[^\p{L}0-9_\-:./]}
+      LEADING_INVALID_CHARS_NO_DIGITS = %r{\A[^\p{L}:]++}
+      LEADING_INVALID_CHARS_WITH_DIGITS = %r{\A[^\p{L}0-9:./\-]++}
+      MAX_BYTE_SIZE = 200
+      MAX_BYTE_SIZE_BUFFER = MAX_BYTE_SIZE * 2
+      TRAILING_UNDERSCORES = %r{_++\z}
+      VALID_ASCII_TAG = %r{\A[a-z:][a-z0-9:./-]*\z}
+
+      # Based on https://github.com/DataDog/datadog-agent/blob/45799c842bbd216bcda208737f9f11cade6fdd95/pkg/trace/traceutil/normalize.go#L131
+      # Specifically:
+      # - Must be valid UTF-8
+      # - Invalid characters are replaced with an underscore
+      # - Leading non-letter characters are removed but colons are kept
+      # - Trailing non-letter characters are removed
+      # - Trailing underscores are removed
+      # - Consecutive underscores are merged into a single underscore
+      # - Maximum length is 200 characters
+      # If it's a tag value, allow it to start with a digit
+      def self.normalize(original_value, remove_digit_start_char: false)
+        transformed_value = original_value.to_s.encode('UTF-8', invalid: :replace, undef: :replace)
+        transformed_value.strip!
+        return "" if transformed_value.empty?
+
+        return transformed_value if transformed_value.bytesize <= MAX_BYTE_SIZE &&
+          transformed_value.match?(VALID_ASCII_TAG)
+
+        if transformed_value.ascii_only? && transformed_value.length <= MAX_BYTE_SIZE
+          normalized_value = transformed_value
+        else
+          byte_position = 0
+          character_count = 0
+          normalized_value = String.new(encoding: 'UTF-8')
+
+          transformed_value.each_char do |char|
+            byte_width = char.bytesize
+            break if byte_position + byte_width > MAX_BYTE_SIZE
+            break if character_count >= MAX_BYTE_SIZE
+
+            normalized_value << char
+            byte_position += byte_width
+            character_count += 1
+          end
+        end
+
+        normalized_value.downcase!
+        normalized_value.gsub!(INVALID_TAG_CHARACTERS, '_')
+
+        # The Trace Agent allows tag values to start with a number so this logic is here too
+        leading_invalid_regex = remove_digit_start_char ? LEADING_INVALID_CHARS_NO_DIGITS : LEADING_INVALID_CHARS_WITH_DIGITS
+        normalized_value.sub!(leading_invalid_regex, "")
+
+        normalized_value.squeeze!('_') if normalized_value.include?('__')
+        normalized_value.sub!(TRAILING_UNDERSCORES, "")
+
+        normalized_value
+      end
+    end
+  end
+end

lib/datadog/tracing/configuration/ext.rb

@@ -15,6 +15,7 @@ module Ext
         ENV_NATIVE_SPAN_EVENTS = 'DD_TRACE_NATIVE_SPAN_EVENTS'
         ENV_RESOURCE_RENAMING_ENABLED = 'DD_TRACE_RESOURCE_RENAMING_ENABLED'
         ENV_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT = 'DD_TRACE_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT'
+        ENV_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED = 'DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED'
 
         # @public_api
         module SpanAttributeSchema

lib/datadog/tracing/transport/trace_formatter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../core/environment/identity'
+require_relative '../../core/environment/process'
 require_relative '../../core/environment/socket'
 require_relative '../../core/environment/git'
 require_relative '../../core/git/ext'
@@ -62,6 +63,7 @@ def format!
           tag_apm_tracing_disabled!
 
           if first_span
+            tag_process_tags!
             tag_git_repository_url!
             tag_git_commit_sha!
           end
@@ -215,6 +217,15 @@ def tag_git_commit_sha!
           first_span.set_tag(Core::Git::Ext::TAG_COMMIT_SHA, git_commit_sha)
         end
 
+        def tag_process_tags!
+          return unless Datadog.configuration.experimental_propagate_process_tags_enabled
+
+          first_span.set_tag(
+            Core::Environment::Ext::TAG_PROCESS_TAGS,
+            Core::Environment::Process.serialized
+          )
+        end
+
         private
 
         def partial?

sig/datadog/core/environment/ext.rbs

@@ -37,6 +37,18 @@ module Datadog
         TAG_SERVICE: String
 
         TAG_VERSION: String
+
+        PROCESS_TYPE: ::String
+
+        TAG_ENTRYPOINT_BASEDIR: ::String
+
+        TAG_ENTRYPOINT_NAME: ::String
+
+        TAG_ENTRYPOINT_WORKDIR: ::String
+
+        TAG_ENTRYPOINT_TYPE: ::String
+
+        TAG_PROCESS_TAGS: ::String
       end
     end
   end

sig/datadog/core/environment/process.rbs

@@ -0,0 +1,21 @@
+module Datadog
+  module Core
+    module Environment
+      module Process
+        @serialized: ::String
+
+        def serialized: () -> ::String
+
+        private
+
+        def entrypoint_workdir: () -> ::String
+
+        def entrypoint_type: () -> ::String
+
+        def entrypoint_name: () -> ::String
+
+        def entrypoint_basedir: () -> ::String
+      end
+    end
+  end
+end

sig/datadog/core/normalizer.rbs

@@ -0,0 +1,15 @@
+module Datadog
+  module Core
+    module Normalizer
+      INVALID_TAG_CHARACTERS: ::Regexp
+      LEADING_INVALID_CHARS_NO_DIGITS: ::Regexp
+      LEADING_INVALID_CHARS_WITH_DIGITS: ::Regexp
+      TRAILING_UNDERSCORES: ::Regexp
+      MAX_BYTE_SIZE: ::Integer
+      MAX_BYTE_SIZE_BUFFER: ::Integer
+      VALID_ASCII_TAG: ::Regexp
+
+      def self.normalize: (untyped original_value, ?remove_digit_start_char: bool) -> ::String
+    end
+  end
+end

spec/datadog/core/configuration/settings_spec.rb

@@ -1337,6 +1337,45 @@
     end
   end
 
+  describe '#experimental_propagate_process_tags_enabled' do
+    subject(:experimental_propagate_process_tags_enabled) { settings.experimental_propagate_process_tags_enabled }
+
+    context "when #{Datadog::Core::Environment::Ext::ENV_VERSION}" do
+      around do |example|
+        ClimateControl.modify('DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED' => environment) do
+          example.run
+        end
+      end
+
+      context 'by default' do
+        let(:environment) { nil }
+
+        it { is_expected.to be false }
+      end
+
+      context 'when set to true' do
+        let(:environment) { 'true' }
+
+        it { is_expected.to be true }
+      end
+
+      context 'when set to false' do
+        let(:environment) { 'false' }
+
+        it { is_expected.to be false }
+      end
+    end
+  end
+
+  describe '#experimental_propagate_process_tags_enabled=' do
+    it 'updates the #experimental_propagate_process_tags_enabled setting' do
+      expect { settings.experimental_propagate_process_tags_enabled = true }
+        .to change { settings.experimental_propagate_process_tags_enabled }
+        .from(false)
+        .to(true)
+    end
+  end
+
   describe '#time_now_provider=' do
     subject(:set_time_now_provider) { settings.time_now_provider = time_now_provider }