Move to own extension

Author: gyuheon0h

Rakefile

@@ -55,16 +55,16 @@ namespace :test do
 
       candidates.each_key do |group|
         env = if group.empty?
-          {}
-        else
-          gemfile = AppraisalConversion.to_bundle_gemfile(group)
-          {'BUNDLE_GEMFILE' => gemfile}
-        end
+                {}
+              else
+                gemfile = AppraisalConversion.to_bundle_gemfile(group)
+                { 'BUNDLE_GEMFILE' => gemfile }
+              end
         command = "bundle check || bundle install && bundle exec rake #{spec_task}"
         command += "'[#{spec_arguments}]'" if spec_arguments
 
-        total_executors = ENV.key?('CIRCLE_NODE_TOTAL') ? ENV['CIRCLE_NODE_TOTAL'].to_i : nil
-        current_executor = ENV.key?('CIRCLE_NODE_INDEX') ? ENV['CIRCLE_NODE_INDEX'].to_i : nil
+        total_executors = Datadog::DATADOG_ENV.key?('CIRCLE_NODE_TOTAL') ? Datadog::DATADOG_ENV['CIRCLE_NODE_TOTAL'].to_i : nil
+        current_executor = Datadog::DATADOG_ENV.key?('CIRCLE_NODE_INDEX') ? Datadog::DATADOG_ENV['CIRCLE_NODE_INDEX'].to_i : nil
 
         if total_executors && current_executor && total_executors > 1
           @execution_count ||= 0
@@ -82,10 +82,10 @@ desc 'Run RSpec'
 namespace :spec do
   # REMINDER: If adding a new task here, make sure also add it to the `Matrixfile`
   task all: [:main, :benchmark, :custom_cop,
-    :graphql, :graphql_unified_trace_patcher, :graphql_trace_patcher, :graphql_tracing_patcher,
-    :rails, :railsredis, :railsredis_activesupport, :railsactivejob,
-    :elasticsearch, :http, :redis, :sidekiq, :sinatra, :hanami, :hanami_autoinstrument,
-    :profiling, :core_with_libdatadog_api, :error_tracking, :open_feature]
+             :graphql, :graphql_unified_trace_patcher, :graphql_trace_patcher, :graphql_tracing_patcher,
+             :rails, :railsredis, :railsredis_activesupport, :railsactivejob,
+             :elasticsearch, :http, :redis, :sidekiq, :sinatra, :hanami, :hanami_autoinstrument,
+             :profiling, :core_with_libdatadog_api, :error_tracking, :open_feature]
 
   desc '' # "Explicitly hiding from `rake -T`"
   RSpec::Core::RakeTask.new(:main) do |t, args|
@@ -228,7 +228,12 @@ namespace :spec do
     t.pattern = CORE_WITH_LIBDATADOG_API.join(', ')
     t.rspec_opts = args.to_a.join(' ')
   end.tap do |t|
-    Rake::Task[t.name].enhance(["compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"])
+    Rake::Task[t.name].enhance(
+      [
+        "compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}",
+        "compile:datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}",
+      ]
+    )
   end
   # rubocop:enable Style/MultilineBlockChain
 
@@ -318,11 +323,18 @@ namespace :spec do
     rescue => e
       # Compilation failed (likely unsupported Ruby version) - tests will skip gracefully
       puts "Warning: libdatadog_api compilation failed: #{e.class}: #{e}"
-      puts "DSM tests will be skipped for this Ruby version"
+      puts 'DSM tests will be skipped for this Ruby version'
+    end
+
+    task :compile_runtime_stacks do
+      Rake::Task["compile:datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"].invoke
+    rescue => e
+      puts "Warning: datadog_runtime_stacks compilation failed: #{e.class}: #{e}"
+      puts 'Runtime stack dependent tests will proceed without native runtime stacks support'
     end
 
     DSM_ENABLED_LIBRARIES.each do |task_name|
-      Rake::Task["spec:#{task_name}"].enhance([:compile_libdatadog_for_dsm])
+      Rake::Task["spec:#{task_name}"].enhance([:compile_libdatadog_for_dsm, :compile_runtime_stacks])
     end
   end
 
@@ -461,11 +473,11 @@ namespace :coverage do
   task :report do
     require 'simplecov'
 
-    resultset_files = Dir["#{ENV.fetch("COVERAGE_DIR", "coverage")}/.resultset.json"] +
-      Dir["#{ENV.fetch("COVERAGE_DIR", "coverage")}/versions/**/.resultset.json"]
+    resultset_files = Dir["#{Datadog::DATADOG_ENV.fetch('COVERAGE_DIR', 'coverage')}/.resultset.json"] +
+      Dir["#{Datadog::DATADOG_ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/**/.resultset.json"]
 
     SimpleCov.collate resultset_files do
-      coverage_dir "#{ENV.fetch("COVERAGE_DIR", "coverage")}/report"
+      coverage_dir "#{Datadog::DATADOG_ENV.fetch('COVERAGE_DIR', 'coverage')}/report"
       formatter SimpleCov::Formatter::HTMLFormatter
     end
   end
@@ -475,11 +487,11 @@ namespace :coverage do
     require 'simplecov'
     require_relative 'spec/support/simplecov_fix'
 
-    versions = Dir["#{ENV.fetch("COVERAGE_DIR", "coverage")}/versions/*"].map { |f| File.basename(f) }
+    versions = Dir["#{Datadog::DATADOG_ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/*"].map { |f| File.basename(f) }
     versions.map do |version|
       puts "Generating report for: #{version}"
-      SimpleCov.collate Dir["#{ENV.fetch("COVERAGE_DIR", "coverage")}/versions/#{version}/**/.resultset.json"] do
-        coverage_dir "#{ENV.fetch("COVERAGE_DIR", "coverage")}/report/versions/#{version}"
+      SimpleCov.collate Dir["#{Datadog::DATADOG_ENV.fetch('COVERAGE_DIR', 'coverage')}/versions/#{version}/**/.resultset.json"] do
+        coverage_dir "#{Datadog::DATADOG_ENV.fetch('COVERAGE_DIR', 'coverage')}/report/versions/#{version}"
         formatter SimpleCov::Formatter::HTMLFormatter
       end
     end
@@ -499,6 +511,10 @@ NATIVE_EXTS = [
     ext.ext_dir = 'ext/libdatadog_api'
   end,
 
+  Rake::ExtensionTask.new("datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}") do |ext|
+    ext.ext_dir = 'ext/datadog_runtime_stacks'
+  end,
+
   Rake::ExtensionTask.new("datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
     ext.ext_dir = 'ext/datadog_profiling_native_extension'
   end,

datadog.gemspec

@@ -83,6 +83,7 @@ Gem::Specification.new do |spec|
 
   spec.extensions = [
     'ext/datadog_profiling_native_extension/extconf.rb',
+    'ext/datadog_runtime_stacks/extconf.rb',
     'ext/libdatadog_api/extconf.rb'
   ]
 end

ext/datadog_runtime_stacks/extconf.rb

@@ -0,0 +1,149 @@
+# rubocop:disable Style/StderrPuts
+# rubocop:disable Style/GlobalVars
+
+require_relative "../libdatadog_extconf_helpers"
+
+SKIPPED_REASON_FILE = "#{__dir__}/skipped_reason.txt".freeze
+# Not a problem if the file doesn't exist or we can't delete it
+begin
+  File.delete(SKIPPED_REASON_FILE)
+rescue
+  nil
+end
+
+def skip_building_extension!(reason)
+  $stderr.puts(
+    "WARN: Skipping build of datadog_runtime_stacks (#{reason}). Runtime stack collection will not be available."
+  )
+
+  fail_install_if_missing_extension = ENV["DD_FAIL_INSTALL_IF_MISSING_EXTENSION"].to_s.strip.downcase == "true"
+
+  if fail_install_if_missing_extension
+    require "mkmf"
+    Logging.message("[datadog] Failure cause: #{reason}")
+  else
+    File.write("Makefile", "all install clean: # dummy makefile that does nothing")
+  end
+
+  exit
+end
+
+if ENV["DD_NO_EXTENSION"].to_s.strip.downcase == "true"
+  skip_building_extension!("the `DD_NO_EXTENSION` environment variable is/was set to `true` during installation")
+end
+skip_building_extension!("current Ruby VM is not supported") if RUBY_ENGINE != "ruby"
+skip_building_extension!("Microsoft Windows is not supported") if Gem.win_platform?
+
+libdatadog_issue = Datadog::LibdatadogExtconfHelpers.load_libdatadog_or_get_issue
+skip_building_extension!("issue setting up `libdatadog` gem: #{libdatadog_issue}") if libdatadog_issue
+
+require "mkmf"
+
+# Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
+# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
+append_cflags "-Werror" if ENV["DATADOG_GEM_CI"] == "true"
+
+# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
+# * by upstream Ruby -- search for gnu99 in the codebase
+# * by msgpack, another datadog gem dependency
+append_cflags "-std=gnu99"
+
+# Gets really noisy when we include the MJIT header, let's omit it
+append_cflags "-Wno-unused-function"
+
+# Allow defining variables at any point in a function
+append_cflags "-Wno-declaration-after-statement"
+
+# If we forget to include a Ruby header, the function call may still appear to work, but then
+# cause a segfault later. Let's ensure that never happens.
+append_cflags "-Werror-implicit-function-declaration"
+
+# The native extension is not intended to expose any symbols/functions for other native libraries to use;
+append_cflags "-fvisibility=hidden"
+
+# Avoid legacy C definitions
+append_cflags "-Wold-style-definition"
+
+# Enable all other compiler warnings
+append_cflags "-Wall"
+append_cflags "-Wextra"
+
+if ENV["DDTRACE_DEBUG"] == "true"
+  $defs << "-DDD_DEBUG"
+  CONFIG["optflags"] = "-O0"
+  CONFIG["debugflags"] = "-ggdb3"
+end
+
+# If we got here, libdatadog is available and loaded
+ENV["PKG_CONFIG_PATH"] = "#{ENV["PKG_CONFIG_PATH"]}:#{Libdatadog.pkgconfig_folder}"
+Logging.message("[datadog] PKG_CONFIG_PATH set to #{ENV["PKG_CONFIG_PATH"].inspect}\n")
+$stderr.puts("Using libdatadog #{Libdatadog::VERSION} from #{Libdatadog.pkgconfig_folder}")
+
+unless pkg_config("datadog_profiling_with_rpath")
+  Logging.message("[datadog] Ruby detected the pkg-config command is #{$PKGCONFIG.inspect}\n")
+
+  if Datadog::LibdatadogExtconfHelpers.pkg_config_missing?
+    skip_building_extension!("the `pkg-config` system tool is missing")
+  else
+    skip_building_extension!("there was a problem in setting up the `libdatadog` dependency")
+  end
+end
+
+# Add Ruby version compatibility defines
+$defs << "-DNO_INT_FIRST_LINENO" if RUBY_VERSION < "3.2"
+$defs << "-DNO_RACTORS" if RUBY_VERSION < "3"
+$defs << "-DNO_RACTOR_HEADER_INCLUDE" if RUBY_VERSION < "3.3"
+
+EXTENSION_NAME = "datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}".freeze
+
+# The MJIT header was introduced on 2.6 and removed on 3.3
+CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
+
+if CAN_USE_MJIT_HEADER
+  mjit_header_file_name = "rb_mjit_min_header-#{RUBY_VERSION}.h"
+
+  # Validate that the mjit header can actually be compiled on this system
+  original_common_headers = MakeMakefile::COMMON_HEADERS
+  MakeMakefile::COMMON_HEADERS = "".freeze
+  unless have_macro("RUBY_MJIT_H", mjit_header_file_name)
+    skip_building_extension!("MJIT header compilation failed")
+  end
+  MakeMakefile::COMMON_HEADERS = original_common_headers
+
+  $defs << "-DRUBY_MJIT_HEADER='\"#{mjit_header_file_name}\"'"
+
+  create_header
+  append_cflags "-Wunused-parameter"
+  create_makefile EXTENSION_NAME
+else
+  # For Rubies without MJIT header, use datadog-ruby_core_source gem
+  create_header
+
+  require "datadog/ruby_core_source"
+  dir_config("ruby") # allow user to pass in non-standard core include directory
+
+  # Workaround for mkmf issue with with_cppflags
+  Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
+    super("#{newflags} #{$CPPFLAGS}", &block)
+  end
+
+  Datadog::RubyCoreSource
+    .create_makefile_with_core(
+      proc do
+        headers_available =
+          have_header("vm_core.h") &&
+          have_header("iseq.h") &&
+          (RUBY_VERSION < "3.3" || have_header("ractor_core.h"))
+
+        if headers_available
+          append_cflags "-Wunused-parameter"
+        end
+
+        headers_available
+      end,
+      EXTENSION_NAME,
+    )
+end
+
+# rubocop:enable Style/GlobalVars
+# rubocop:enable Style/StderrPuts