[8.19] (backport #18536) Use gradle and bundled jruby for acceptance tests orchestration

build.gradle

@@ -696,7 +696,24 @@ tasks.register("runIntegrationTests") {
     shouldRunAfter ":logstash-core:test"
 }
 
+tasks.register("installAcceptanceTestGems", Exec) {
+    dependsOn bootstrap
+    inputs.file("${projectDir}/qa/Gemfile")
+    outputs.file("${projectDir}/qa/Gemfile.lock")
+    workingDir "${projectDir}/qa"
+    environment "BUNDLE_PATH", "vendor/bundle"
+    commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "install"
+}
 
+tasks.register("runAcceptanceTests", Exec) {
+    dependsOn installAcceptanceTestGems
+    workingDir "${projectDir}/qa"
+    environment "BUNDLE_PATH", "vendor/bundle"
+    // NOTE: We sub-process here to avoid issues in java 21+ with Jruby whereby access to some modules that
+    // are required for file descriptors to be properly inherited when ruby does sub-processing.
+    // https://github.com/elastic/logstash/pull/18536#discussion_r2655013858
+    commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rake", "qa:acceptance:all"
+}
 
 tasks.register("generateLicenseReport", JavaExec) {
     dependsOn generateLicenseReportInputs

ci/acceptance_tests.sh

@@ -52,11 +52,5 @@ else
   esac
 fi
 
-echo "--- Acceptance: Installing dependencies"
-cd $QA_DIR
-bundle install
-
 echo "--- Acceptance: Running the tests"
-# TODO: figure out how to encapsulate all this in gradle or at least ensure the bundler
-# used here ends up using the runtime managed with LS. 
-rake qa:acceptance:all
+./gradlew runAcceptanceTests

qa/rspec/commands/base.rb

@@ -59,7 +59,8 @@ class InstallException < Exception; end
   class Base
     LOCATION = ENV.fetch('LS_ARTIFACTS_PATH', LS_BUILD_PATH.freeze)
     LOGSTASH_PATH = "/usr/share/logstash/".freeze
-
+    # Clear Bundler/Ruby environment variables that could pollute tested Logstash instance
+    ENV_CLEANUP = "env -u BUNDLE_PATH -u BUNDLE_GEMFILE -u BUNDLE_BIN_PATH -u BUNDLER_VERSION -u BUNDLER_SETUP -u GEM_HOME -u GEM_PATH -u RUBYLIB -u RUBYOPT".freeze
     def start_service(service)
       service_manager(service, "start")
     end
@@ -69,9 +70,7 @@ def stop_service(service)
     end
 
     def run_command(cmd)
-      response = nil
-      response = sudo_exec!("JARS_SKIP='true' #{cmd}")
-      response
+      sudo_exec!("#{ENV_CLEANUP} JARS_SKIP='true' #{cmd}")
     end
 
     def replace_in_gemfile(pattern, replace)