diff --git a/build.gradle b/build.gradle index a238e899729..a89552fa617 100644 --- a/build.gradle +++ b/build.gradle @@ -418,6 +418,7 @@ project(":logstash-core") { tasks.getByPath(":logstash-core:" + tsk).configure { dependsOn copyPluginTestAlias dependsOn installDevelopmentGems + dependsOn installDefaultGems } } } @@ -1001,6 +1002,7 @@ if (System.getenv('OSS') != 'true') { ["rubyTests", "rubyIntegrationTests", "test"].each { tsk -> tasks.getByPath(":logstash-xpack:" + tsk).configure { dependsOn installDevelopmentGems + dependsOn installDefaultGems } } } diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index f1c93cff45c..9403875c7b3 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -20,6 +20,10 @@ module Bundler extend self def patch! + + return if @bundler_patched + @bundler_patched = true + # Patch to prevent Bundler to save a .bundle/config file in the root # of the application ::Bundler::Settings.module_exec do @@ -40,15 +44,15 @@ def self.reset_paths! end end - # When preparing offline packs or generally when installing gems, bundler wants to have `.gem` files + # When preparing offline packs or generally when installing gems, bundler wants to have `.gem` files # cached. We ship a default set of gems that inclue all of the unpacked code. During dependency - # resolution bundler still wants to ensure`.gem` files exist. This patch updates two paths in bundler where + # resolution bundler still wants to ensure`.gem` files exist. This patch updates two paths in bundler where # it natively it would *fail* when a `.gem` file is not found. Instead of failing we force the cache to be # updated with a `.gem` file. This preserves the original patch behavior. There is still an open question of - # *how* to potentially update the files we vendor or the way we set up bundler to avoid carrying this patch. + # *how* to potentially update the files we vendor or the way we set up bundler to avoid carrying this patch. # As of JRuby 9.4.13.0 rubygems (bundler) is at 3.6.3. There have been some releases and changes in bundler code # since then but it does not seem to have changed the way it handles gem files. Obviously carrying a patch like this - # carries a maintenance burden so prioritizing a packaging solution may be + # carries a maintenance burden so prioritizing a packaging solution may be ::Bundler::Source::Rubygems.module_exec do def fetch_gem_if_possible(spec, previous_spec = nil) path = if spec.remote @@ -75,18 +79,304 @@ def cache(spec, custom_path = nil) raise InstallError, e.message end end + + # + # BACKPORT: Fix `--prefer-local` flag (from rubygems/rubygems commits 607a3bf479, 209b93a, 23047a0) + # + # The original implementation of --prefer-local was too naive: + # 1. It didn't pass prefer_local to Package objects + # 2. It returned empty array when no local specs exist (instead of falling back to remote) + # 3. It didn't properly handle default gems + # + # These patches fix: + # - PR #7951: Fix `--prefer-local` flag (propagate to packages, add fallback logic) + # - PR #8412: Fix `--prefer-local` not respecting default gems + # - PR #8484: Fix `bundle install --prefer-local` sometimes installing very old versions + # + + # Patch Source base class to add prefer_local! method + ::Bundler::Source.class_eval do + def prefer_local! + # Base implementation - does nothing, subclasses override + end + end + + # Patch Source::Rubygems to track prefer_local state and handle default_specs properly + # Also add support for JRuby bundled gems (gems in vendor/jruby/.../specifications/) + ::Bundler::Source::Rubygems.class_eval do + # Add prefer_local! method + def prefer_local! + @prefer_local = true + end + + # Return specs from JRuby's bundled gem directory (specifications/, not specifications/default/) + # These are gems that ship with JRuby but aren't "default gems" in the Ruby sense + def jruby_bundled_specs + @jruby_bundled_specs ||= begin + idx = ::Bundler::Index.new + jruby_gem_home = LogStash::Bundler.instance_variable_get(:@jruby_default_gem_dir) + jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir) + jruby_default_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_default_specs_dir) + + if jruby_gem_home && jruby_specs_dir && ::File.directory?(jruby_specs_dir) + # gems_dir is where the actual gem code lives (gem_home/gems/) + jruby_gems_dir = ::File.join(jruby_gem_home, "gems") + + # Get gemspecs from specifications/ but NOT from specifications/default/ + ::Dir[::File.join(jruby_specs_dir, "*.gemspec")].each do |path| + # Skip if this is actually in the default directory + next if jruby_default_specs_dir && path.start_with?(jruby_default_specs_dir) + + # gemspec_stub params: filename, base_dir, gems_dir + # base_dir = gem home (parent of specifications/) + # gems_dir = where gem code lives (gem_home/gems/) + stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_gem_home, jruby_gems_dir) + # Create a Bundler::StubSpecification from the Gem::StubSpecification + bundler_spec = ::Bundler::StubSpecification.from_stub(stub) + # Set source to self (the Source::Rubygems instance) - required for materialization + bundler_spec.source = self + idx << bundler_spec + end + end + idx + end + end + + # Override specs method to handle prefer_local for default_specs AND jruby_bundled_specs + alias_method :original_specs, :specs + + def specs + @specs ||= begin + # remote_specs usually generates a way larger Index than the other + # sources, and large_idx.merge! small_idx is way faster than + # small_idx.merge! large_idx. + index = @allow_remote ? remote_specs.dup : ::Bundler::Index.new + index.merge!(cached_specs) if @allow_cached + index.merge!(installed_specs) if @allow_local + + if @allow_local + if @prefer_local + # With prefer_local, merge jruby_bundled_specs and default_specs so they take precedence + # over remote/cached/installed specs. This ensures JRuby's bundled gems are preferred. + index.merge!(jruby_bundled_specs) + index.merge!(default_specs) + else + # complete with default specs, only if not already available in the + # index through remote, cached, or installed specs + index.use(jruby_bundled_specs) + index.use(default_specs) + end + end + + index + end + end + end + + # Patch SourceList to propagate prefer_local! to all sources + ::Bundler::SourceList.class_eval do + def prefer_local! + all_sources.each(&:prefer_local!) + end + end + + # Patch Definition to call sources.prefer_local! when prefer_local! is called + ::Bundler::Definition.class_eval do + alias_method :original_prefer_local!, :prefer_local! + + def prefer_local! + @prefer_local = true + sources.prefer_local! + end + end + + # Patch Package to add prefer_local support + ::Bundler::Resolver::Package.class_eval do + def prefer_local? + @prefer_local + end + + def consider_remote_versions! + @prefer_local = false + end + end + + # Patch Resolver::Base to propagate prefer_local to packages and add include_remote_specs + ::Bundler::Resolver::Base.class_eval do + alias_method :original_base_initialize, :initialize + + def initialize(source_requirements, dependencies, base, platforms, options) + @prefer_local_option = options[:prefer_local] + original_base_initialize(source_requirements, dependencies, base, platforms, options) + end + + alias_method :original_get_package, :get_package + + def get_package(name) + package = original_get_package(name) + # Inject prefer_local into packages since older Bundler doesn't pass it through + if @prefer_local_option && !package.instance_variable_get(:@prefer_local) + package.instance_variable_set(:@prefer_local, true) + end + package + end + + def include_remote_specs(names) + names.each do |name| + get_package(name).consider_remote_versions! + end + end + end + + # Patch Resolver to fix filter_remote_specs with proper fallback + ::Bundler::Resolver.class_eval do + # Override filter_remote_specs with the fixed version from Bundler 2.7+ + # This fixes the issue where --prefer-local would return empty specs + # when no local gems are installed, instead of falling back to remote + def filter_remote_specs(specs, package) + if package.prefer_local? + local_specs = specs.select {|s| s.is_a?(::Bundler::StubSpecification) } + + if local_specs.empty? + # BACKPORT FIX: If no local specs exist, fall back to remote specs + # instead of returning empty array + package.consider_remote_versions! + specs + else + local_specs + end + else + specs + end + end + end + + # Patch Source::Rubygems#install to skip installation for default gems and JRuby bundled gems + # The original condition `spec.default_gem? && !cached_built_in_gem(...)` has a side effect: + # cached_built_in_gem fetches from remote if not in cache. For default gems and JRuby bundled gems, + # we should skip installation entirely without needing a cached .gem file. + ::Bundler::Source::Rubygems.class_eval do + alias_method :original_rubygems_install, :install + + def install(spec, options = {}) + # For default gems, skip installation entirely - they're already available + if spec.default_gem? + print_using_message "Using #{version_message(spec, options[:previous_spec])}" + return nil + end + + # For JRuby bundled gems, also skip installation - they're already available + # Check if this exact gem (name + version) exists in JRuby's bundled gems + jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir) + if jruby_specs_dir && ::File.directory?(jruby_specs_dir) + jruby_gemspec_path = ::File.join(jruby_specs_dir, "#{spec.name}-#{spec.version}.gemspec") + if ::File.exist?(jruby_gemspec_path) + print_using_message "Using #{version_message(spec, options[:previous_spec])}" + return nil + end + end + + original_rubygems_install(spec, options) + end + end end + # Capture JRuby's default gem directory before paths are changed + # This is needed so that default gems (like json) can be found with --prefer-local + def preserve_jruby_default_gems_path + return @jruby_default_gem_dir if defined?(@jruby_default_gem_dir) + + # The Gradle/JRuby setup already changes Gem.default_dir to a temp path before + # this code runs, so we need to construct the actual JRuby path from LOGSTASH_HOME + logstash_home = ENV["LOGSTASH_HOME"] || ::File.expand_path("../../..", __FILE__) + jruby_gems_dir = ::File.join(logstash_home, "vendor", "jruby", "lib", "ruby", "gems", "shared") + jruby_default_specs = ::File.join(jruby_gems_dir, "specifications", "default") + jruby_bundled_specs = ::File.join(jruby_gems_dir, "specifications") + + if ::File.directory?(jruby_default_specs) + @jruby_default_gem_dir = jruby_gems_dir + @jruby_default_specs_dir = jruby_default_specs + @jruby_bundled_specs_dir = jruby_bundled_specs + else + # Fall back to Gem.default_dir if vendor/jruby doesn't exist + @jruby_default_gem_dir = ::Gem.default_dir + @jruby_default_specs_dir = ::Gem.default_specifications_dir + @jruby_bundled_specs_dir = nil + end + + @jruby_default_gem_dir + end + + # Patch Gem::Specification.default_stubs to also look in JRuby's original specs directories + # This is needed because Gem.default_specifications_dir only returns a single path, + # and after Gem.paths = ENV it points to Logstash's gem home, not JRuby's installation + # We include BOTH: + # - specifications/default/ (true default gems) + # - specifications/ (JRuby bundled gems like rexml, rake, net-imap, etc.) + def patch_default_stubs! + return if @default_stubs_patched || !defined?(@jruby_default_specs_dir) || @jruby_default_specs_dir.nil? + @default_stubs_patched = true + + jruby_default_specs_dir = @jruby_default_specs_dir + jruby_bundled_specs_dir = @jruby_bundled_specs_dir + jruby_gem_home = @jruby_default_gem_dir + + ::Gem::Specification.singleton_class.class_eval do + alias_method :original_default_stubs, :default_stubs + + define_method(:default_stubs) do |pattern = "*.gemspec"| + # Get stubs from the current default_specifications_dir + stubs = original_default_stubs(pattern) + + # Also look in JRuby's original default specs directory (specifications/default/) + if jruby_default_specs_dir && ::File.directory?(jruby_default_specs_dir) && jruby_default_specs_dir != ::Gem.default_specifications_dir + ::Dir[::File.join(jruby_default_specs_dir, pattern)].each do |path| + # Use default_gemspec_stub to mark these as default gems (default_gem = true) + stub = ::Gem::StubSpecification.default_gemspec_stub(path, jruby_default_specs_dir, jruby_default_specs_dir) + stubs << stub unless stubs.any? { |s| s.name == stub.name && s.version == stub.version } + end + end + + # Also include JRuby's bundled gems (specifications/, excluding specifications/default/) + # These are gems like rexml, rake, net-imap that ship with JRuby but aren't "default gems" + # We treat them as default gems here so they're available via Bundler's add_default_gems_to + if jruby_bundled_specs_dir && jruby_gem_home && ::File.directory?(jruby_bundled_specs_dir) + jruby_gems_dir = ::File.join(jruby_gem_home, "gems") + ::Dir[::File.join(jruby_bundled_specs_dir, pattern)].each do |path| + # Skip if this is in the default directory (already handled above) + next if jruby_default_specs_dir && path.start_with?(jruby_default_specs_dir) + + # Use gemspec_stub (not default_gemspec_stub) with correct base_dir and gems_dir + stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_gem_home, jruby_gems_dir) + stubs << stub unless stubs.any? { |s| s.name == stub.name && s.version == stub.version } + end + end + + stubs + end + end + end + # prepare bundler's environment variables, but do not invoke ::Bundler::setup def prepare(options = {}) options = {:without => [:development]}.merge(options) options[:without] = Array(options[:without]) + # Capture JRuby default gems path BEFORE clearing + jruby_gem_dir = preserve_jruby_default_gems_path + ::Gem.clear_paths - ENV['GEM_HOME'] = ENV['GEM_PATH'] = Environment.logstash_gem_home + # Include both Logstash gem home AND JRuby's default gem directory in GEM_PATH + # This ensures default gems can be discovered by Gem::Specification.default_stubs + gem_path = [Environment.logstash_gem_home, jruby_gem_dir].compact.uniq.join(::File::PATH_SEPARATOR) + ENV['GEM_HOME'] = Environment.logstash_gem_home + ENV['GEM_PATH'] = gem_path ::Gem.paths = ENV + # Patch default_stubs to also look in JRuby's original location + patch_default_stubs! + # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s) ENV["BUNDLE_GEMFILE"] = Environment::GEMFILE_PATH @@ -127,9 +417,21 @@ def invoke!(options = {}) :jobs => 12, :all => false, :package => false, :without => [:development]}.merge(options) options[:without] = Array(options[:without]) options[:update] = Array(options[:update]) if options[:update] + + # Capture JRuby default gems path BEFORE clearing + jruby_gem_dir = preserve_jruby_default_gems_path + ::Gem.clear_paths - ENV['GEM_HOME'] = ENV['GEM_PATH'] = LogStash::Environment.logstash_gem_home + # Include both Logstash gem home AND JRuby's default gem directory in GEM_PATH + # This ensures default gems can be discovered by Gem::Specification.default_stubs + gem_path = [LogStash::Environment.logstash_gem_home, jruby_gem_dir].compact.uniq.join(::File::PATH_SEPARATOR) + ENV['GEM_HOME'] = LogStash::Environment.logstash_gem_home + ENV['GEM_PATH'] = gem_path ::Gem.paths = ENV + + # Patch default_stubs to also look in JRuby's original location + patch_default_stubs! + # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s). # in the context of calling Bundler::CLI this is not really required since Bundler::CLI will look at # Bundler.settings[:gemfile] unlike Bundler.setup. For the sake of consistency and defensive/future proofing, let's keep it here. @@ -280,6 +582,7 @@ def bundler_arguments(options = {}) arguments = [] if options[:install] arguments << "install" + arguments << "--prefer-local" arguments << "--clean" if options[:clean] if options[:local] arguments << "--local" diff --git a/lib/bootstrap/rspec.rb b/lib/bootstrap/rspec.rb index 782455135ee..db6cdeb7571 100755 --- a/lib/bootstrap/rspec.rb +++ b/lib/bootstrap/rspec.rb @@ -17,6 +17,16 @@ require_relative "environment" LogStash::Bundler.setup!({:without => [:build]}) +# Our use of LogStash::Bundler.setup! here leaves us in kind of a wonky state for *all* tests +# Essentially we end up with a load path that favors bundlers gem env over stdlib. This is +# not really the call stack in logstash itself, so while this does make the full bundled gem +# env available for tests, it also has a quirk where stdlib gems are not loaed correctly. The +# following patch ensures that stdlib gems are bumped to the front of the load path for unit +# tests. +## START PATCH ## +jruby_stdlib = $LOAD_PATH.find { |p| p.end_with?('vendor/jruby/lib/ruby/stdlib') } +$LOAD_PATH.unshift($LOAD_PATH.delete(jruby_stdlib)) if jruby_stdlib +## END PATCH ## require "logstash-core" require "logstash/environment" diff --git a/lib/pluginmanager/bundler/logstash_uninstall.rb b/lib/pluginmanager/bundler/logstash_uninstall.rb index 996e840cf27..1897fed5b8b 100644 --- a/lib/pluginmanager/bundler/logstash_uninstall.rb +++ b/lib/pluginmanager/bundler/logstash_uninstall.rb @@ -35,6 +35,7 @@ def initialize(gemfile_path, lockfile_path) end def uninstall!(gems_to_remove) + gems_to_remove = Array(gems_to_remove) unsatisfied_dependency_mapping = Dsl.evaluate(gemfile_path, lockfile_path, {}).specs.each_with_object({}) do |spec, memo| diff --git a/lib/pluginmanager/command.rb b/lib/pluginmanager/command.rb index 5a7c1a51f97..70d7d680a47 100644 --- a/lib/pluginmanager/command.rb +++ b/lib/pluginmanager/command.rb @@ -50,9 +50,14 @@ def remove_unused_locally_installed_gems! def remove_orphan_dependencies! locked_gem_names = ::Bundler::LockfileParser.new(File.read(LogStash::Environment::LOCKFILE)).specs.map(&:full_name).to_set + bundle_path = LogStash::Environment::BUNDLE_DIR + # JRuby bundled gems path - never touch these + jruby_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "jruby", "lib", "ruby", "gems") orphan_gem_specs = ::Gem::Specification.each .reject(&:stubbed?) # skipped stubbed (uninstalled) gems .reject(&:default_gem?) # don't touch jruby-included default gems + .reject { |spec| spec.full_gem_path.start_with?(jruby_gems_path) } # don't touch jruby bundled gems + .select { |spec| spec.full_gem_path.start_with?(bundle_path) } # only gems in bundle path .reject{ |spec| locked_gem_names.include?(spec.full_name) } .sort diff --git a/lib/pluginmanager/gem_installer.rb b/lib/pluginmanager/gem_installer.rb index e5560a78025..17eb889ef8e 100644 --- a/lib/pluginmanager/gem_installer.rb +++ b/lib/pluginmanager/gem_installer.rb @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +require "bootstrap/environment" require "pluginmanager/ui" require "pathname" require "rubygems/package" @@ -26,17 +27,16 @@ module LogStash module PluginManager # - Generate the specifications # - Copy the data in the right folders class GemInstaller - GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "3.1.0")) SPECIFICATIONS_DIR = "specifications" GEMS_DIR = "gems" CACHE_DIR = "cache" attr_reader :gem_home - def initialize(gem_file, display_post_install_message = false, gem_home = GEM_HOME) + def initialize(gem_file, display_post_install_message = false) @gem_file = gem_file @gem = ::Gem::Package.new(@gem_file) - @gem_home = Pathname.new(gem_home) + @gem_home = Pathname.new(LogStash::Environment.logstash_gem_home) @display_post_install_message = display_post_install_message end @@ -48,8 +48,8 @@ def install post_install_message end - def self.install(gem_file, display_post_install_message = false, gem_home = GEM_HOME) - self.new(gem_file, display_post_install_message, gem_home).install + def self.install(gem_file, display_post_install_message = false) + self.new(gem_file, display_post_install_message).install end private diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 2739e1fc6a7..27fd91d44bb 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -101,18 +101,6 @@ namespace "artifact" do @exclude_paths << 'vendor/**/gems/**/Gemfile.lock' @exclude_paths << 'vendor/**/gems/**/Gemfile' - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/rake-*' - # exclude ruby-maven-libs 3.3.9 jars until JRuby ships with >= 3.8.9 - @exclude_paths << 'vendor/bundle/jruby/**/gems/ruby-maven-libs-3.3.9/**/*' - - # remove this after JRuby includes rexml 3.3.x - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/rexml-3.2.5/**/*' - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/specifications/rexml-3.2.5.gemspec' - - # remove this after JRuby includes net-imap-0.2.4+ - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/specifications/net-imap-0.2.3.gemspec' - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/net-imap-0.2.3/**/*' - @exclude_paths.freeze end diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 47d572f417a..d37ca541d01 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -90,12 +90,65 @@ namespace "plugin" do task.reenable # Allow this task to be run again end # task "install" + task "clean-duplicate-gems" do + shared_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME, + 'vendor/jruby/lib/ruby/gems/shared/gems') + default_gemspecs_path = File.join(LogStash::Environment::LOGSTASH_HOME, + 'vendor/jruby/lib/ruby/gems/shared/specifications/default') + bundle_gems_path = File.join(LogStash::Environment::BUNDLE_DIR, + 'jruby/*/gems') + + # "bundled" gems in jruby + # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L108-L152 + shared_gem_names = Dir.glob(File.join(shared_gems_path, '*')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + # "default" gems in jruby/ruby + # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L21-L106 + default_gem_names = Dir.glob(File.join(default_gemspecs_path, '*.gemspec')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + # gems we explicitly manage with bundler (we always want these to take precedence) + bundle_gem_names = Dir.glob(File.join(bundle_gems_path, '*')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + shared_duplicates = shared_gem_names & bundle_gem_names + default_duplicates = default_gem_names & bundle_gem_names + all_duplicates = (shared_duplicates + default_duplicates).uniq + + puts("[plugin:clean-duplicate-gems] Removing duplicate gems: #{all_duplicates.sort.join(', ')}") + + # Remove shared/bundled gem duplicates + shared_duplicates.each do |gem_name| + FileUtils.rm_rf(Dir.glob("#{shared_gems_path}/#{gem_name}-*")) + FileUtils.rm_rf(Dir.glob("#{shared_gems_path}/../specifications/#{gem_name}-*.gemspec")) + end + + # Remove default gem gemspecs only + default_duplicates.each do |gem_name| + # For stdlib default gems we only remove the gemspecs as removing the source code + # files results in code loading errors and ruby warnings + FileUtils.rm_rf(Dir.glob("#{default_gemspecs_path}/#{gem_name}-*.gemspec")) + end + + task.reenable + end + + task "install-default" => "bootstrap" do puts("[plugin:install-default] Installing default plugins") remove_lockfile # because we want to use the release lockfile install_plugins("--no-verify", "--preserve", *LogStash::RakeLib::DEFAULT_PLUGINS) + # Clean duplicates after full gem resolution + Rake::Task["plugin:clean-duplicate-gems"].invoke task.reenable # Allow this task to be run again end diff --git a/rubyUtils.gradle b/rubyUtils.gradle index 4e2565f72a4..0eb244ae96c 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -176,7 +176,7 @@ void setupJruby(File projectDir, File buildDir) { executeJruby projectDir, buildDir, { ScriptingContainer jruby -> jruby.currentDirectory = projectDir jruby.runScriptlet("require '${projectDir}/lib/bootstrap/environment'") - jruby.runScriptlet("LogStash::Bundler.invoke!") + jruby.runScriptlet("LogStash::Bundler.invoke!(:install => true)") jruby.runScriptlet("LogStash::Bundler.genericize_platform") } } @@ -190,12 +190,13 @@ void setupJruby(File projectDir, File buildDir) { Object executeJruby(File projectDir, File buildDir, Closure /* Object*/ block) { def jruby = new ScriptingContainer() def env = jruby.environment - def gemDir = "${projectDir}/vendor/bundle/jruby/3.1.0".toString() + def gemHomeDir = "${projectDir}/vendor/bundle/jruby/3.1.0".toString() + def gemPathDir = "${projectDir}/vendor/jruby/lib/ruby/gems/shared:${projectDir}/vendor/bundle/jruby/3.1.0".toString() jruby.setLoadPaths(["${projectDir}/vendor/jruby/lib/ruby/stdlib".toString()]) env.put "USE_RUBY", "1" - env.put "GEM_HOME", gemDir + env.put "GEM_HOME", gemHomeDir env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString() - env.put "GEM_PATH", gemDir + env.put "GEM_PATH", gemPathDir // Pass through ORG_GRADLE_PROJECT_fedrampHighMode if it exists in the project properties // See https://docs.gradle.org/current/userguide/build_environment.html#setting_a_project_property // For more information about setting properties via env vars prefixed with ORG_GRADLE_PROJECT @@ -279,7 +280,11 @@ tasks.register("installCustomJRuby", Copy) { dependsOn buildCustomJRuby description = "Install custom built JRuby in the vendor directory" inputs.file(customJRubyTar) - outputs.dir("${projectDir}/vendor/jruby") + // Don't re-extract if core JRuby is already installed. This works around + // gem deduplication when rake calls back in to gradle. + onlyIf { + !file("${projectDir}/vendor/jruby/bin/jruby").exists() + } from tarTree(customJRubyTar == "" ? jrubyTarPath : customJRubyTar) eachFile { f -> f.path = f.path.replaceFirst("^jruby-${customJRubyVersion}", '') @@ -294,7 +299,11 @@ tasks.register("downloadAndInstallJRuby", Copy) { dependsOn=[verifyFile, installCustomJRuby] description = "Install JRuby in the vendor directory" inputs.file(jrubyTarPath) - outputs.dir("${projectDir}/vendor/jruby") + // Don't re-extract if core JRuby is already installed. This works around + // gem deduplication when rake calls back in to gradle. + onlyIf { + !file("${projectDir}/vendor/jruby/bin/jruby").exists() + } from tarTree(downloadJRuby.dest) eachFile { f -> f.path = f.path.replaceFirst("^jruby-${jRubyVersion}", '') diff --git a/spec/unit/bootstrap/bundler_spec.rb b/spec/unit/bootstrap/bundler_spec.rb index 44b65207d7a..b0d5f0d1b64 100644 --- a/spec/unit/bootstrap/bundler_spec.rb +++ b/spec/unit/bootstrap/bundler_spec.rb @@ -62,7 +62,9 @@ expect(::Bundler.settings[:gemfile]).to eq(LogStash::Environment::GEMFILE_PATH) expect(::Bundler.settings[:without]).to eq(options.fetch(:without, [])) - expect(ENV['GEM_PATH']).to eq(LogStash::Environment.logstash_gem_home) + # GEM_PATH includes both logstash gem home and JRuby's gem directory + # so that JRuby bundled gems (like rexml) can be found + expect(ENV['GEM_PATH']).to start_with(LogStash::Environment.logstash_gem_home) $stderr = original_stderr end diff --git a/spec/unit/plugin_manager/gem_installer_spec.rb b/spec/unit/plugin_manager/gem_installer_spec.rb index dd85009dfff..89ee9c93a51 100644 --- a/spec/unit/plugin_manager/gem_installer_spec.rb +++ b/spec/unit/plugin_manager/gem_installer_spec.rb @@ -27,18 +27,27 @@ let(:simple_gem) { ::File.join(::File.dirname(__FILE__), "..", "..", "support", "pack", "valid-pack", "logstash", "valid-pack", "#{plugin_name}.gem") } subject { described_class } - let(:temporary_gem_home) { p = Stud::Temporary.pathname; FileUtils.mkdir_p(p); p } + let(:gem_home) { LogStash::Environment.logstash_gem_home } + # Clean up installed gems after each test + after(:each) do + spec_file = ::File.join(gem_home, "specifications", "#{plugin_name}.gemspec") + FileUtils.rm_f(spec_file) if ::File.exist?(spec_file) + gem_dir = ::File.join(gem_home, "gems", plugin_name) + FileUtils.rm_rf(gem_dir) if Dir.exist?(gem_dir) + cache_file = ::File.join(gem_home, "cache", "#{plugin_name}.gem") + FileUtils.rm_f(cache_file) if ::File.exist?(cache_file) + end it "install the specifications in the spec dir" do - subject.install(simple_gem, false, temporary_gem_home) - spec_file = ::File.join(temporary_gem_home, "specifications", "#{plugin_name}.gemspec") + subject.install(simple_gem, false) + spec_file = ::File.join(gem_home, "specifications", "#{plugin_name}.gemspec") expect(::File.exist?(spec_file)).to be_truthy expect(::File.size(spec_file)).to be > 0 end it "install the gem in the gems dir" do - subject.install(simple_gem, false, temporary_gem_home) - gem_dir = ::File.join(temporary_gem_home, "gems", plugin_name) + subject.install(simple_gem, false) + gem_dir = ::File.join(gem_home, "gems", plugin_name) expect(Dir.exist?(gem_dir)).to be_truthy end @@ -50,13 +59,13 @@ context "when we want the message" do it "display the message" do - expect(subject.install(simple_gem, true, temporary_gem_home)).to eq(message) + expect(subject.install(simple_gem, true)).to eq(message) end end context "when we dont want the message" do it "doesn't display the message" do - expect(subject.install(simple_gem, false, temporary_gem_home)).to be_nil + expect(subject.install(simple_gem, false)).to be_nil end end end @@ -65,7 +74,7 @@ context "when we don't want the message" do it "doesn't display the message" do expect(LogStash::PluginManager.ui).not_to receive(:info).with(message) - subject.install(simple_gem, true, temporary_gem_home) + subject.install(simple_gem, true) end end end