diff --git a/.github/workflows/specs.yml b/.github/workflows/specs.yml index d08cf6a7..b2dbe62e 100644 --- a/.github/workflows/specs.yml +++ b/.github/workflows/specs.yml @@ -2,30 +2,19 @@ name: Specs on: - push - pull_request +permissions: + contents: read jobs: - all_specs: - name: All Specs + specs: + name: Specs strategy: matrix: - ruby: ['3.1', '3.2', '3.3', '3.4'] - gemfile: ['Gemfile'] + ruby: ['3.2', '3.3', '3.4'] runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Start LocalStack - run: docker compose up -d - - - name: Wait for LocalStack - run: | - timeout 30s bash -c ' - until curl -s http://localhost:4566/_localstack/health | grep -q "\"sqs\": \"available\""; do - echo "Waiting for LocalStack..." - sleep 2 - done - ' - - uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0 with: ruby-version: ${{ matrix.ruby }} @@ -33,51 +22,44 @@ jobs: - name: Run specs run: bundle exec rake spec - env: - BUNDLE_GEMFILE: ${{ matrix.gemfile }} - - - name: Run integration specs - run: bundle exec rake spec:integration - env: - BUNDLE_GEMFILE: ${{ matrix.gemfile }} - rails_specs: - name: Rails Specs + integrations: + name: Integrations strategy: matrix: - rails: ['7.2', '8.0', '8.1'] - include: - - rails: '7.2' - ruby: '3.2' - gemfile: gemfiles/rails_7_2.gemfile - - rails: '8.0' - ruby: '3.3' - gemfile: gemfiles/rails_8_0.gemfile - - rails: '8.1' - ruby: '3.4' - gemfile: gemfiles/rails_8_1.gemfile + ruby: ['3.2', '3.3', '3.4'] runs-on: ubuntu-latest - env: - BUNDLE_GEMFILE: ${{ matrix.gemfile }} steps: - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Start LocalStack + run: docker compose up -d + + - name: Wait for LocalStack + run: | + timeout 30s bash -c ' + until curl -s http://localhost:4566/_localstack/health | grep -q "\"sqs\": \"available\""; do + echo "Waiting for LocalStack..." + sleep 2 + done + ' + - uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0 with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - name: Run Rails specs - run: bundle exec rake spec:rails + - name: Run integration specs + run: bundle exec rake spec:integration ci-success: name: CI Success runs-on: ubuntu-latest if: always() needs: - - all_specs - - rails_specs + - specs + - integrations steps: - name: Check all jobs passed if: | diff --git a/.gitignore b/.gitignore index 50b9bddd..a9e8afb9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ shoryuken.yml rubocop.html .byebug_history .localstack +spec/integration/**/Gemfile.lock +spec/integration/**/vendor/ +spec/integration/**/.bundle/ diff --git a/.rspec b/.rspec index 83e16f80..a7d85e81 100644 --- a/.rspec +++ b/.rspec @@ -1,2 +1,3 @@ --color --require spec_helper +--exclude-pattern "spec/integration/**/*" diff --git a/CHANGELOG.md b/CHANGELOG.md index 76e32079..f557b069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,35 @@ ## [7.0.0] - Unreleased +- Enhancement: Add `enqueue_all` for bulk ActiveJob enqueuing (Rails 7.1+) + - Implements efficient bulk enqueuing using SQS `send_message_batch` API + - Called by `ActiveJob.perform_all_later` for batching multiple jobs + - Batches jobs in groups of 10 (SQS limit) per queue + - Groups jobs by queue name for efficient multi-queue handling + - Enhancement: Add ActiveJob Continuations support (Rails 8.1+) - Implements `stopping?` method in ActiveJob adapters to signal graceful shutdown - Enables jobs to checkpoint progress and resume after interruption - Handles past timestamps correctly (SQS treats negative delays as immediate delivery) - Tracks shutdown state in Launcher via `stopping?` flag - Leverages existing Shoryuken shutdown lifecycle (stop/stop! methods) - - Includes comprehensive integration tests with continuable jobs - See Rails PR #55127 for more details on ActiveJob Continuations +- Enhancement: Add CurrentAttributes persistence support + - Enables Rails `ActiveSupport::CurrentAttributes` to flow from enqueue to job execution + - Automatically serializes current attributes into job payload when enqueuing + - Restores attributes before job execution and resets them afterward + - Supports multiple CurrentAttributes classes + - Based on Sidekiq's approach using `ActiveJob::Arguments` for serialization + - Usage: `require 'shoryuken/active_job/current_attributes'` and + `Shoryuken::ActiveJob::CurrentAttributes.persist('MyApp::Current')` + +- Breaking: Drop support for Ruby 3.1 (EOL March 2025) + - Minimum required Ruby version is now 3.2.0 + - Supported Ruby versions: 3.2, 3.3, 3.4 + - Users on Ruby 3.1 should upgrade or remain on Shoryuken 6.x + - Breaking: Remove support for Rails versions older than 7.2 - - Rails 7.0 and 7.1 have reached end-of-life and are no longer supported - - Supported versions: Rails 7.2, 8.0, and 8.1 + - Rails 7.0 and 7.1 have reached end-of-life (April 2025) and are no longer supported + - Supported Rails versions: 7.2, 8.0, and 8.1 - Users on older Rails versions should upgrade or remain on Shoryuken 6.x - Enhancement: Replace Concurrent::AtomicFixnum with pure Ruby AtomicCounter diff --git a/Gemfile b/Gemfile index b5b3667a..733110c0 100644 --- a/Gemfile +++ b/Gemfile @@ -12,7 +12,6 @@ group :test do end group :development do - gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git' gem 'pry-byebug' gem 'rubocop' end diff --git a/Rakefile b/Rakefile index 03f0b3e6..2769360f 100644 --- a/Rakefile +++ b/Rakefile @@ -5,19 +5,13 @@ $stdout.sync = true begin require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new(:spec) do |t| - t.exclude_pattern = 'spec/integration/**/*_spec.rb' - end + RSpec::Core::RakeTask.new(:spec) namespace :spec do - desc 'Run Rails specs only' - RSpec::Core::RakeTask.new(:rails) do |t| - t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb' - end - desc 'Run integration specs only' - RSpec::Core::RakeTask.new(:integration) do |t| - t.pattern = 'spec/integration/**/*_spec.rb' + task :integration do + puts "Running integration tests..." + system('./bin/integrations') || exit(1) end end rescue LoadError diff --git a/bin/clean_localstack b/bin/clean_localstack new file mode 100755 index 00000000..ab99bc36 --- /dev/null +++ b/bin/clean_localstack @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Removes all integration test SQS queues from LocalStack +# +# Useful when having a long-running LocalStack instance that cannot be fully +# restarted between test runs. All integration test queues use the 'it-' prefix, +# making them easy to identify and remove. +# +# Usage: +# bin/clean_localstack + +require 'aws-sdk-sqs' + +THREADS_COUNT = 3 + +sqs = Aws::SQS::Client.new( + region: 'us-east-1', + endpoint: 'http://localhost:4566', + access_key_id: 'fake', + secret_access_key: 'fake' +) + +# Find all queues with 'it-' prefix +response = sqs.list_queues(queue_name_prefix: 'it-') +queues_for_removal = response.queue_urls || [] + +if queues_for_removal.empty? + puts "No integration test queues found (prefix: it-)" + exit 0 +end + +puts "Found #{queues_for_removal.size} queues to remove" + +queue = SizedQueue.new(THREADS_COUNT) + +threads = Array.new(THREADS_COUNT) do + Thread.new do + while (queue_url = queue.pop) + queue_name = queue_url.split('/').last + puts "Removing queue: #{queue_name}" + sqs.delete_queue(queue_url: queue_url) + end + end +end + +queues_for_removal.each { |url| queue << url } + +queue.close +threads.each(&:join) + +puts "Cleanup complete" diff --git a/bin/integrations b/bin/integrations new file mode 100755 index 00000000..17cacc64 --- /dev/null +++ b/bin/integrations @@ -0,0 +1,267 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Shoryuken integration test runner +# +# Usage: +# bin/integrations # Run all integration tests +# bin/integrations fifo # Run tests with 'fifo' in path +# bin/integrations rails/rails_72 # Run Rails 7.2 tests +# bin/integrations batch retry # Run tests matching 'batch' OR 'retry' +# bin/integrations -v fifo # Run with verbose output + +require 'fileutils' +require 'timeout' + +TIMEOUT = 300 # 5 minutes per scenario +SPEC_DIR = File.expand_path('../spec/integration', __dir__) +ROOT_DIR = File.expand_path('..', __dir__) + +class IntegrationRunner + def initialize(args) + @verbose = args.delete('-v') || args.delete('--verbose') + @filters = args.reject { |a| a.start_with?('-') } + end + + def run + specs = find_specs + specs = filter_specs(specs) if @filters.any? + + if specs.empty? + puts 'No specs found matching filters' + exit 1 + end + + puts "Running #{specs.size} integration specs..." + puts + + results = run_specs(specs) + report_results(results) + end + + private + + def find_specs + Dir.glob(File.join(SPEC_DIR, '**/*_spec.rb')).reject do |path| + # Exclude vendor and .bundle directories + path.include?('/vendor/') || path.include?('/.bundle/') + end.map do |path| + relative_path = path.sub("#{SPEC_DIR}/", '') + dir = File.dirname(path) + gemfile = File.exist?(File.join(dir, 'Gemfile')) ? File.join(dir, 'Gemfile') : File.join(ROOT_DIR, 'Gemfile') + + { + name: relative_path.sub('_spec.rb', '').gsub('/', ' / '), + path: path, + relative_path: relative_path, + directory: dir, + gemfile: gemfile + } + end.sort_by { |s| s[:relative_path] } + end + + def filter_specs(specs) + specs.select do |spec| + @filters.any? { |filter| spec[:relative_path].include?(filter) } + end + end + + def run_specs(specs) + results = [] + + specs.each do |spec| + result = run_spec(spec) + results << result + + if result[:skipped] + print 'S' + elsif result[:success] + print '.' + else + print 'F' + end + $stdout.flush + end + + puts + results + end + + def run_spec(spec) + env = { + 'BUNDLE_GEMFILE' => spec[:gemfile], + 'RAILS_ENV' => 'test' + } + + # Install dependencies if using a local Gemfile + uses_local_gemfile = spec[:gemfile] != File.join(ROOT_DIR, 'Gemfile') + if uses_local_gemfile + install_result = install_bundle(spec, env) + unless install_result[:success] + # Skip test if bundle install fails (e.g., gems not available in CI) + return { + spec: spec, + success: true, + skipped: true, + skip_reason: 'Bundle install failed (dependencies not available)', + output: install_result[:output] + } + end + + # Use isolated bundle config to match install_bundle + bundle_path = File.join(spec[:directory], 'vendor', 'bundle') + bundle_config = File.join(spec[:directory], '.bundle') + env['BUNDLE_PATH'] = bundle_path + env['BUNDLE_FROZEN'] = 'false' + env['BUNDLE_APP_CONFIG'] = bundle_config + end + + # Run the spec + # For local gemfiles, use standalone bundle setup which doesn't need bundler at runtime + # This avoids issues with bundle exec inheriting the wrong config + cmd = if uses_local_gemfile + standalone_setup = File.join(bundle_path, 'bundler', 'setup.rb') + ['ruby', "-r#{standalone_setup}", File.join(ROOT_DIR, 'bin/scenario'), spec[:path]] + else + ['bundle', 'exec', 'ruby', File.join(ROOT_DIR, 'bin/scenario'), spec[:path]] + end + + output = [] + start_time = Time.now + + begin + Timeout.timeout(TIMEOUT) do + IO.popen(env, cmd, chdir: spec[:directory], err: [:child, :out]) do |io| + io.each_line { |line| output << line } + end + end + + { + spec: spec, + success: $?.success?, + exit_code: $?.exitstatus, + duration: Time.now - start_time, + output: output.join + } + rescue Timeout::Error + { + spec: spec, + success: false, + exit_code: -1, + error: 'Timeout', + duration: Time.now - start_time, + output: output.join + } + end + end + + def install_bundle(spec, env) + return { success: true } if @bundle_installed&.include?(spec[:gemfile]) + + output = [] + + # Create isolated bundle environment to avoid CI cache interference + # Use a unique path per Gemfile to avoid conflicts + bundle_path = File.join(spec[:directory], 'vendor', 'bundle') + bundle_config = File.join(spec[:directory], '.bundle') + + # Create local .bundle/config to override project-level config + FileUtils.mkdir_p(bundle_config) + File.write(File.join(bundle_config, 'config'), <<~CONFIG) + --- + BUNDLE_PATH: "#{bundle_path}" + BUNDLE_FROZEN: "false" + CONFIG + + clean_env = env.merge( + 'BUNDLE_PATH' => bundle_path, + 'BUNDLE_FROZEN' => 'false', + 'BUNDLE_DEPLOYMENT' => nil, + 'BUNDLE_WITHOUT' => nil, + 'BUNDLE_CACHE_PATH' => nil, + 'BUNDLE_BIN' => nil, + 'BUNDLE_APP_CONFIG' => bundle_config, + 'RUBYOPT' => nil # Clear any -rbundler/setup from CI + ) + + # Use --standalone to generate a setup.rb that doesn't need bundler at runtime + # Run in a completely unbundled environment using Bundler API + cmd_script = <<~RUBY + require 'bundler' + Bundler.with_unbundled_env do + system({'BUNDLE_GEMFILE' => '#{spec[:gemfile]}', 'BUNDLE_PATH' => '#{bundle_path}', 'BUNDLE_FROZEN' => 'false', 'BUNDLE_APP_CONFIG' => '#{bundle_config}'}, 'bundle', 'install', '--standalone') + end + exit($?.success? ? 0 : 1) + RUBY + + IO.popen(['ruby', '-e', cmd_script], chdir: spec[:directory], err: [:child, :out]) do |io| + io.each_line { |line| output << line } + end + + @bundle_installed ||= [] + @bundle_installed << spec[:gemfile] if $?.success? + + { + spec: spec, + success: $?.success?, + output: output.join, + error: $?.success? ? nil : 'Bundle install failed', + bundle_config: bundle_config + } + end + + def report_results(results) + skipped = results.select { |r| r[:skipped] } + failed = results.reject { |r| r[:success] || r[:skipped] } + passed = results.count { |r| r[:success] && !r[:skipped] } + total = results.size + + puts + summary = "#{passed}/#{total} passed" + summary += ", #{skipped.size} skipped" if skipped.any? + puts summary + + if skipped.any? + puts + puts 'Skipped:' + puts + skipped.each do |result| + puts " - #{result[:spec][:name]}" + puts " Reason: #{result[:skip_reason]}" if result[:skip_reason] + if result[:output] && !result[:output].strip.empty? + lines = result[:output].lines.last(15) + lines.each { |line| puts " #{line}" } + end + end + end + + if failed.any? + puts + puts 'Failures:' + puts + + failed.each_with_index do |result, idx| + puts " #{idx + 1}) #{result[:spec][:name]}" + if result[:error] + puts " Error: #{result[:error]}" + end + if result[:output] && !result[:output].strip.empty? + # Show last 30 lines of output for context + lines = result[:output].lines + if lines.size > 30 + puts " ... (#{lines.size - 30} lines truncated)" + lines = lines.last(30) + end + lines.each { |line| puts " #{line}" } + end + puts + end + end + + exit(failed.empty? ? 0 : 1) + end +end + +if __FILE__ == $0 + IntegrationRunner.new(ARGV.dup).run +end diff --git a/bin/scenario b/bin/scenario new file mode 100755 index 00000000..1ef6724b --- /dev/null +++ b/bin/scenario @@ -0,0 +1,154 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Individual scenario runner for integration testing +# This script runs a single integration test file in complete isolation + +require 'bundler/setup' + +# Exit codes +EXIT_SUCCESS = 0 +EXIT_FAILURE = 1 +EXIT_TIMEOUT = 2 +EXIT_SETUP_ERROR = 3 + +class ScenarioRunner + attr_reader :test_file + + def initialize(test_file) + @test_file = test_file + @exit_code = EXIT_SUCCESS + end + + def run + puts "Running: #{File.basename(test_file)}" if ENV['VERBOSE'] + + # Set up the scenario-specific environment + setup_scenario + + # Load and run the test file + load_and_run_test + + exit EXIT_SUCCESS + rescue => e + puts "FAILED: #{File.basename(test_file)} - #{e.message}" if ENV['VERBOSE'] + puts e.backtrace.first(5).join("\n") if ENV['VERBOSE'] + exit EXIT_FAILURE + end + + private + + def setup_scenario + # Each test handles its own specific requirements + require 'bundler/setup' + + puts "Setting up isolated test environment" if ENV['VERBOSE'] + end + + + def load_and_run_test + # Test file might be relative to current directory + if File.exist?(test_file) + absolute_test_path = File.expand_path(test_file) + else + # Fallback to project root resolution + project_root = File.expand_path('..', __dir__) + absolute_test_path = File.join(project_root, test_file) + end + + puts "Current directory: #{Dir.pwd}" if ENV['VERBOSE'] + puts "Loading test file: #{absolute_test_path}" if ENV['VERBOSE'] + + unless File.exist?(absolute_test_path) + raise "Test file not found: #{absolute_test_path}" + end + + # Check if this is an RSpec file (contains RSpec.describe or describe) + file_content = File.read(absolute_test_path, encoding: 'UTF-8') + if file_content.match?(/\b(?:RSpec\.describe|describe)\b/) + puts "Running as RSpec test" if ENV['VERBOSE'] + run_rspec_test(absolute_test_path) + else + puts "Running as plain Ruby test" if ENV['VERBOSE'] + # Load integrations_helper for plain Ruby integration tests + if absolute_test_path.include?('spec/integration') + project_root = File.expand_path('..', __dir__) + integrations_helper = File.join(project_root, 'spec', 'integrations_helper.rb') + require integrations_helper + end + # Load as plain Ruby test + load absolute_test_path + end + end + + def run_rspec_test(test_file_path) + # Change to project root for RSpec to find spec_helper + project_root = File.expand_path('..', __dir__) + Dir.chdir(project_root) do + # Disable SimpleCov for integration tests to avoid coverage failures + ENV['SIMPLECOV_DISABLED'] = 'true' + + # Make the test file path relative to project root for RSpec + relative_test_path = test_file_path.sub("#{project_root}/", '') + + puts "Running RSpec with file: #{relative_test_path}" if ENV['VERBOSE'] + puts "Working directory: #{Dir.pwd}" if ENV['VERBOSE'] + + # Check if this test requires Rails but Rails is not available + if requires_rails?(test_file_path) && !rails_available? + puts "Skipping #{File.basename(test_file_path)} - Rails not available" + return + end + + # Run RSpec with the specific test file + require 'rspec/core' + + # Load integration spec_helper for integration tests + if relative_test_path.include?('spec/integration/') + require_relative '../spec/integration/spec_helper' + end + + result = RSpec::Core::Runner.run([relative_test_path], $stderr, $stdout) + + if result != 0 + raise "RSpec failed with exit code #{result}" + end + ensure + # Clean up environment + ENV.delete('SIMPLECOV_DISABLED') + end + end + + def requires_rails?(test_file_path) + # Check if the test file mentions Rails dependencies + content = File.read(test_file_path) + content.match?(/require.*rails|Rails::|ActiveJob::|ActionController::/) + end + + def rails_available? + begin + require 'rails' + true + rescue LoadError + false + end + end +end + +# Validate arguments +if ARGV.empty? + puts "Usage: bin/scenario " + puts "Example: bin/scenario spec/integration/rails_integration_spec.rb" + exit EXIT_SETUP_ERROR +end + +test_file = ARGV[0] + +unless File.exist?(test_file) + puts "Test file not found: #{test_file}" + exit EXIT_SETUP_ERROR +end + + +# Run the scenario +ScenarioRunner.new(test_file).run diff --git a/gemfiles/.gitignore b/gemfiles/.gitignore deleted file mode 100644 index 71afd1cc..00000000 --- a/gemfiles/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.gemfile.lock diff --git a/gemfiles/rails_7_2.gemfile b/gemfiles/rails_7_2.gemfile deleted file mode 100644 index 6f39f0a0..00000000 --- a/gemfiles/rails_7_2.gemfile +++ /dev/null @@ -1,19 +0,0 @@ -# This file was generated by Appraisal - -source "https://rubygems.org" - -group :test do - gem "activejob", "~> 7.2" - gem "httparty" - gem "multi_xml" - gem "simplecov" - gem "warning" -end - -group :development do - gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git" - gem "pry-byebug" - gem "rubocop" -end - -gemspec path: "../" diff --git a/gemfiles/rails_8_0.gemfile b/gemfiles/rails_8_0.gemfile deleted file mode 100644 index 061b03ad..00000000 --- a/gemfiles/rails_8_0.gemfile +++ /dev/null @@ -1,19 +0,0 @@ -# This file was generated by Appraisal - -source "https://rubygems.org" - -group :test do - gem "activejob", "~> 8.0" - gem "httparty" - gem "multi_xml" - gem "simplecov" - gem "warning" -end - -group :development do - gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git" - gem "pry-byebug" - gem "rubocop" -end - -gemspec path: "../" diff --git a/gemfiles/rails_8_1.gemfile b/gemfiles/rails_8_1.gemfile deleted file mode 100644 index e7471ec0..00000000 --- a/gemfiles/rails_8_1.gemfile +++ /dev/null @@ -1,19 +0,0 @@ -# This file was generated by Appraisal - -source "https://rubygems.org" - -group :test do - gem "activejob", "~> 8.1" - gem "httparty" - gem "multi_xml" - gem "simplecov" - gem "warning" -end - -group :development do - gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git" - gem "pry-byebug" - gem "rubocop" -end - -gemspec path: "../" diff --git a/lib/shoryuken/extensions/active_job_extensions.rb b/lib/active_job/extensions.rb similarity index 85% rename from lib/shoryuken/extensions/active_job_extensions.rb rename to lib/active_job/extensions.rb index 66d628d8..e255e260 100644 --- a/lib/shoryuken/extensions/active_job_extensions.rb +++ b/lib/active_job/extensions.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Shoryuken - module ActiveJobExtensions + module ActiveJob # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs # (instances of ActiveJob::Base). Shoryuken ActiveJob queue adapters use # these parameters when enqueueing jobs; other adapters can ignore them. @@ -36,5 +36,5 @@ def enqueue(options = {}) end end -ActiveJob::Base.include Shoryuken::ActiveJobExtensions::SQSSendMessageParametersAccessor -ActiveJob::Base.prepend Shoryuken::ActiveJobExtensions::SQSSendMessageParametersSupport +ActiveJob::Base.include Shoryuken::ActiveJob::SQSSendMessageParametersAccessor +ActiveJob::Base.prepend Shoryuken::ActiveJob::SQSSendMessageParametersSupport \ No newline at end of file diff --git a/lib/shoryuken/extensions/active_job_adapter.rb b/lib/active_job/queue_adapters/shoryuken_adapter.rb similarity index 66% rename from lib/shoryuken/extensions/active_job_adapter.rb rename to lib/active_job/queue_adapters/shoryuken_adapter.rb index fbaac0b7..032ad0ae 100644 --- a/lib/shoryuken/extensions/active_job_adapter.rb +++ b/lib/active_job/queue_adapters/shoryuken_adapter.rb @@ -4,19 +4,25 @@ # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters require 'shoryuken' +require 'shoryuken/active_job/job_wrapper' module ActiveJob module QueueAdapters # == Shoryuken adapter for Active Job # - # Shoryuken ("sho-ryu-ken") is a super-efficient AWS SQS thread based message processor. - # - # Read more about Shoryuken {here}[https://github.com/ruby-shoryuken/shoryuken]. - # # To use Shoryuken set the queue_adapter config to +:shoryuken+. # # Rails.application.config.active_job.queue_adapter = :shoryuken - class ShoryukenAdapter < ActiveJob::QueueAdapters::AbstractAdapter + + # Determine the appropriate base class based on Rails version + # This prevents AbstractAdapter autoloading issues in Rails 7.0-7.1 + base = if defined?(Rails) && defined?(Rails::VERSION) + (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2 ? Object : AbstractAdapter) + else + Object + end + + class ShoryukenAdapter < base class << self def instance # https://github.com/ruby-shoryuken/shoryuken/pull/174#issuecomment-174555657 @@ -65,6 +71,34 @@ def enqueue_at(job, timestamp) # :nodoc: enqueue(job, delay_seconds: calculate_delay(timestamp)) end + # Bulk enqueue multiple jobs efficiently using SQS batch API. + # Called by ActiveJob.perform_all_later (Rails 7.1+). + # + # @param jobs [Array] jobs to enqueue + # @return [Integer] number of jobs successfully enqueued + def enqueue_all(jobs) # :nodoc: + jobs.group_by(&:queue_name).each do |queue_name, queue_jobs| + queue = Shoryuken::Client.queues(queue_name) + + queue_jobs.each_slice(10) do |batch| + entries = batch.map.with_index do |job, idx| + register_worker!(job) + msg = message(queue, job) + job.sqs_send_message_parameters = msg + { id: idx.to_s }.merge(msg) + end + + response = queue.send_messages(entries: entries) + successful_ids = response.successful.map { |r| r.id.to_i }.to_set + batch.each_with_index do |job, idx| + job.successfully_enqueued = successful_ids.include?(idx) + end + end + end + + jobs.count(&:successfully_enqueued?) + end + private def calculate_delay(timestamp) @@ -97,24 +131,12 @@ def message(queue, job) end def register_worker!(job) - Shoryuken.register_worker(job.queue_name, JobWrapper) - end - - class JobWrapper # :nodoc: - include Shoryuken::Worker - - shoryuken_options body_parser: :json, auto_delete: true - - def perform(sqs_msg, hash) - receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i - past_receives = receive_count - 1 - Base.execute hash.merge({ 'executions' => past_receives }) - end + Shoryuken.register_worker(job.queue_name, Shoryuken::ActiveJob::JobWrapper) end MESSAGE_ATTRIBUTES = { 'shoryuken_class' => { - string_value: JobWrapper.to_s, + string_value: Shoryuken::ActiveJob::JobWrapper.to_s, data_type: 'String' } }.freeze diff --git a/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb b/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb similarity index 98% rename from lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb rename to lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb index 06c2b969..0939ec36 100644 --- a/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +++ b/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb @@ -2,6 +2,8 @@ # ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters +require_relative 'shoryuken_adapter' + module ActiveJob module QueueAdapters # == Shoryuken concurrent adapter for Active Job diff --git a/lib/shoryuken.rb b/lib/shoryuken.rb index 22a4afc7..1abb3707 100644 --- a/lib/shoryuken.rb +++ b/lib/shoryuken.rb @@ -1,25 +1,35 @@ # frozen_string_literal: true -require 'yaml' -require 'json' require 'aws-sdk-sqs' +require 'json' +require 'logger' require 'time' require 'concurrent' require 'forwardable' require 'zeitwerk' +require 'yaml' # Set up Zeitwerk loader loader = Zeitwerk::Loader.for_gem -loader.ignore("#{__dir__}/shoryuken/extensions") +loader.ignore("#{__dir__}/active_job") loader.setup module Shoryuken extend SingleForwardable + # Returns the global Shoryuken configuration options instance. + # This is used internally for storing and accessing configuration settings. + # + # @return [Shoryuken::Options] The global options instance def self.shoryuken_options @_shoryuken_options ||= Shoryuken::Options.new end + # Checks if the Shoryuken server is running and healthy. + # A server is considered healthy when all configured processing groups + # are running and able to process messages. + # + # @return [Boolean] true if the server is healthy def self.healthy? Shoryuken::Runner.instance.healthy? end @@ -77,7 +87,7 @@ def self.healthy? end if Shoryuken.active_job? - require 'shoryuken/extensions/active_job_extensions' - require 'shoryuken/extensions/active_job_adapter' - require 'shoryuken/extensions/active_job_concurrent_send_adapter' + require 'active_job/extensions' + require 'active_job/queue_adapters/shoryuken_adapter' + require 'active_job/queue_adapters/shoryuken_concurrent_send_adapter' end diff --git a/lib/shoryuken/active_job/current_attributes.rb b/lib/shoryuken/active_job/current_attributes.rb new file mode 100644 index 00000000..ec55ff4f --- /dev/null +++ b/lib/shoryuken/active_job/current_attributes.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require 'active_support/current_attributes' +require 'active_job' + +module Shoryuken + module ActiveJob + # Middleware to persist Rails CurrentAttributes across job execution. + # + # This ensures that request-scoped context (like current user, tenant, locale) + # automatically flows from the code that enqueues a job to the job's execution. + # + # Based on Sidekiq's approach to persisting current attributes. + # + # @example Setup in initializer + # require 'shoryuken/active_job/current_attributes' + # Shoryuken::ActiveJob::CurrentAttributes.persist('MyApp::Current') + # + # @example Multiple CurrentAttributes classes + # Shoryuken::ActiveJob::CurrentAttributes.persist('MyApp::Current', 'MyApp::RequestContext') + # + # @see https://api.rubyonrails.org/classes/ActiveSupport/CurrentAttributes.html + # @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb + module CurrentAttributes + # Serializer for current attributes using ActiveJob::Arguments. + # Supports Symbols and GlobalID objects. + module Serializer + module_function + + def serialize(attrs) + ::ActiveJob::Arguments.serialize([attrs]).first + end + + def deserialize(attrs) + ::ActiveJob::Arguments.deserialize([attrs]).first + end + end + + class << self + # @return [Hash] registered CurrentAttributes classes mapped to keys + attr_reader :cattrs + + # Register CurrentAttributes classes to persist across job execution. + # + # @param klasses [Array] CurrentAttributes class names or classes + # @example + # Shoryuken::ActiveJob::CurrentAttributes.persist('Current') + # Shoryuken::ActiveJob::CurrentAttributes.persist(Current, RequestContext) + def persist(*klasses) + @cattrs ||= {} + + klasses.flatten.each_with_index do |klass, idx| + key = @cattrs.empty? ? 'cattr' : "cattr_#{idx}" + @cattrs[key] = klass.to_s + end + + # Prepend the persistence module to the adapter for serialization + unless ::ActiveJob::QueueAdapters::ShoryukenAdapter.ancestors.include?(Persistence) + ::ActiveJob::QueueAdapters::ShoryukenAdapter.prepend(Persistence) + end + + # Prepend the loading module to JobWrapper for deserialization + unless Shoryuken::ActiveJob::JobWrapper.ancestors.include?(Loading) + Shoryuken::ActiveJob::JobWrapper.prepend(Loading) + end + end + end + + # Module prepended to ShoryukenAdapter to serialize CurrentAttributes on enqueue. + module Persistence + private + + def message(queue, job) + hash = super + + CurrentAttributes.cattrs&.each do |key, klass_name| + next if hash[:message_body].key?(key) + + klass = klass_name.constantize + attrs = klass.attributes + next if attrs.empty? + + hash[:message_body][key] = Serializer.serialize(attrs) + end + + hash + end + end + + # Module prepended to JobWrapper to restore CurrentAttributes on execute. + module Loading + def perform(sqs_msg, hash) + klasses_to_reset = [] + + CurrentAttributes.cattrs&.each do |key, klass_name| + next unless hash.key?(key) + + klass = klass_name.constantize + klasses_to_reset << klass + + begin + attrs = Serializer.deserialize(hash[key]) + attrs.each do |attr_name, value| + klass.public_send(:"#{attr_name}=", value) if klass.respond_to?(:"#{attr_name}=") + end + rescue => e + # Log but don't fail if attributes can't be restored + # (e.g., attribute removed between enqueue and execute) + Shoryuken.logger.warn("Failed to restore CurrentAttributes #{klass_name}: #{e.message}") + end + end + + super + ensure + klasses_to_reset.each(&:reset) + end + end + end + end +end diff --git a/lib/shoryuken/active_job/job_wrapper.rb b/lib/shoryuken/active_job/job_wrapper.rb new file mode 100644 index 00000000..022b6591 --- /dev/null +++ b/lib/shoryuken/active_job/job_wrapper.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'active_job' +require 'shoryuken/worker' + +module Shoryuken + module ActiveJob + # Internal worker class that processes ActiveJob jobs. + # This class bridges ActiveJob's interface with Shoryuken's worker interface. + # + # @api private + class JobWrapper # :nodoc: + include Shoryuken::Worker + + shoryuken_options body_parser: :json, auto_delete: true + + # Processes an ActiveJob job from an SQS message. + # + # @param sqs_msg [Shoryuken::Message] The SQS message containing the job data + # @param hash [Hash] The parsed job data from the message body + def perform(sqs_msg, hash) + receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i + past_receives = receive_count - 1 + ::ActiveJob::Base.execute hash.merge({ 'executions' => past_receives }) + end + end + end +end diff --git a/lib/shoryuken/environment_loader.rb b/lib/shoryuken/environment_loader.rb index 8f8ca5d0..7cee517d 100644 --- a/lib/shoryuken/environment_loader.rb +++ b/lib/shoryuken/environment_loader.rb @@ -80,9 +80,9 @@ def initialize_rails end end if Shoryuken.active_job? - require 'shoryuken/extensions/active_job_extensions' - require 'shoryuken/extensions/active_job_adapter' - require 'shoryuken/extensions/active_job_concurrent_send_adapter' + require 'active_job/extensions' + require 'active_job/queue_adapters/shoryuken_adapter' + require 'active_job/queue_adapters/shoryuken_concurrent_send_adapter' end require File.expand_path('config/environment.rb') end diff --git a/lib/shoryuken/util.rb b/lib/shoryuken/util.rb index 1ca34070..3f1e30e0 100644 --- a/lib/shoryuken/util.rb +++ b/lib/shoryuken/util.rb @@ -35,7 +35,7 @@ def worker_name(worker_class, sqs_msg, body = nil) && sqs_msg.message_attributes \ && sqs_msg.message_attributes['shoryuken_class'] \ && sqs_msg.message_attributes['shoryuken_class'][:string_value] \ - == ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper.to_s \ + == 'Shoryuken::ActiveJob::JobWrapper' \ && body "ActiveJob/#{body['job_class']}" diff --git a/renovate.json b/renovate.json index 921dfd47..e8b7f699 100644 --- a/renovate.json +++ b/renovate.json @@ -7,5 +7,23 @@ "github-actions": { "enabled": true, "pinDigests": true - } + }, + "bundler": { + "enabled": true, + "fileMatch": ["(^|/)Gemfile$", "\\.gemfile$", "(^|/)gems\\.rb$", "spec/gemfiles/.+\\.gemfile$", "spec/integration/.*/Gemfile$"] + }, + "packageRules": [ + { + "matchManagers": ["bundler"], + "matchFiles": ["spec/gemfiles/**"], + "groupName": "Rails test dependencies", + "description": "Group Rails version-specific test Gemfiles together" + }, + { + "matchManagers": ["bundler"], + "matchFiles": ["spec/integration/**/Gemfile"], + "groupName": "Integration test dependencies", + "description": "Group integration test Gemfiles together" + } + ] } diff --git a/shoryuken.gemspec b/shoryuken.gemspec index 3d845ef0..2e7f88fe 100644 --- a/shoryuken.gemspec +++ b/shoryuken.gemspec @@ -26,5 +26,5 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'rake' spec.add_development_dependency 'rspec' - spec.required_ruby_version = '>= 3.1.0' + spec.required_ruby_version = '>= 3.2.0' end diff --git a/spec/integration/.rspec b/spec/integration/.rspec new file mode 100644 index 00000000..c99d2e73 --- /dev/null +++ b/spec/integration/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/spec/integration/active_job/adapter_configuration/configuration_spec.rb b/spec/integration/active_job/adapter_configuration/configuration_spec.rb new file mode 100644 index 00000000..714e2608 --- /dev/null +++ b/spec/integration/active_job/adapter_configuration/configuration_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# This spec tests ActiveJob adapter configuration including adapter type, +# Rails 7.2+ transaction commit hook, and singleton pattern. + +setup_active_job + +class ConfigTestJob < ActiveJob::Base + queue_as :config_test + + def perform(data) + "Processed: #{data}" + end +end + +adapter = ActiveJob::Base.queue_adapter +assert_equal("ActiveJob::QueueAdapters::ShoryukenAdapter", adapter.class.name) + +adapter_instance = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert(adapter_instance.respond_to?(:enqueue_after_transaction_commit?)) +assert_equal(true, adapter_instance.enqueue_after_transaction_commit?) + +instance1 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +instance2 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +assert_equal(instance1.object_id, instance2.object_id) +assert(instance1.is_a?(ActiveJob::QueueAdapters::ShoryukenAdapter)) diff --git a/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb b/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb new file mode 100644 index 00000000..47549d05 --- /dev/null +++ b/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Bulk enqueue integration test +# Tests perform_all_later with the new enqueue_all method using SQS batch API + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name) + +class BulkTestJob < ActiveJob::Base + def perform(index, data) + DT[:executions] << { + index: index, + data: data, + job_id: job_id, + executed_at: Time.now + } + end +end + +BulkTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +jobs = (1..15).map { |i| BulkTestJob.new(i, "payload_#{i}") } + +# Use perform_all_later which should call enqueue_all +ActiveJob.perform_all_later(jobs) + +successfully_enqueued_count = jobs.count(&:successfully_enqueued?) +assert_equal(15, successfully_enqueued_count, "Expected all 15 jobs to be marked as successfully enqueued") + +poll_queues_until(timeout: 45) do + DT[:executions].size >= 15 +end + +assert_equal(15, DT[:executions].size, "Expected 15 job executions, got #{DT[:executions].size}") + +executed_indices = DT[:executions].map { |e| e[:index] }.sort +expected_indices = (1..15).to_a +assert_equal(expected_indices, executed_indices, "All job indices should be present") + +DT[:executions].each do |execution| + expected_data = "payload_#{execution[:index]}" + assert_equal(expected_data, execution[:data], "Job #{execution[:index]} should have correct data") +end + +job_ids = DT[:executions].map { |e| e[:job_id] } +assert_equal(15, job_ids.uniq.size, "All job IDs should be unique") diff --git a/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb b/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb new file mode 100644 index 00000000..7d35a06e --- /dev/null +++ b/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# CurrentAttributes are persisted correctly when using bulk enqueue (perform_all_later) + +setup_localstack +setup_active_job + +require 'active_support/current_attributes' +require 'shoryuken/active_job/current_attributes' + +queue_name = DT.queue +create_test_queue(queue_name) + +class TestCurrent < ActiveSupport::CurrentAttributes + attribute :user_id, :tenant_id +end + +Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent) + +class BulkCurrentAttributesTestJob < ActiveJob::Base + def perform(index) + DT[:executions] << { + index: index, + user_id: TestCurrent.user_id, + tenant_id: TestCurrent.tenant_id + } + end +end + +BulkCurrentAttributesTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +TestCurrent.user_id = 'bulk-user-123' +TestCurrent.tenant_id = 'bulk-tenant' + +jobs = (1..3).map { |i| BulkCurrentAttributesTestJob.new(i) } +ActiveJob.perform_all_later(jobs) + +TestCurrent.reset + +poll_queues_until(timeout: 30) { DT[:executions].size >= 3 } + +assert_equal(3, DT[:executions].size) +DT[:executions].each do |job| + assert_equal('bulk-user-123', job[:user_id]) + assert_equal('bulk-tenant', job[:tenant_id]) +end diff --git a/spec/integration/active_job/current_attributes/complex_types_spec.rb b/spec/integration/active_job/current_attributes/complex_types_spec.rb new file mode 100644 index 00000000..c9fea688 --- /dev/null +++ b/spec/integration/active_job/current_attributes/complex_types_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# CurrentAttributes with complex data types (hashes, arrays, symbols) are serialized and restored + +setup_localstack +setup_active_job + +require 'active_support/current_attributes' +require 'shoryuken/active_job/current_attributes' + +queue_name = DT.queue +create_test_queue(queue_name) + +class TestCurrent < ActiveSupport::CurrentAttributes + attribute :user_id, :tenant_id +end + +Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent) + +class ComplexTypesTestJob < ActiveJob::Base + def perform + DT[:executions] << { + user_id: TestCurrent.user_id, + tenant_id: TestCurrent.tenant_id + } + end +end + +ComplexTypesTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +TestCurrent.user_id = { role: :admin, permissions: [:read, :write, :delete] } +TestCurrent.tenant_id = [:tenant_a, :tenant_b] + +ComplexTypesTestJob.perform_later + +TestCurrent.reset + +poll_queues_until(timeout: 30) { DT[:executions].size >= 1 } + +result = DT[:executions].first + +user_data = result[:user_id] +assert(user_data.is_a?(Hash)) +role = user_data['role'] || user_data[:role] +assert_equal('admin', role.to_s) +permissions = user_data['permissions'] || user_data[:permissions] +assert_equal(3, permissions.size) + +tenant_data = result[:tenant_id] +assert(tenant_data.is_a?(Array)) +assert_equal(2, tenant_data.size) diff --git a/spec/integration/active_job/current_attributes/empty_context_spec.rb b/spec/integration/active_job/current_attributes/empty_context_spec.rb new file mode 100644 index 00000000..89e9134f --- /dev/null +++ b/spec/integration/active_job/current_attributes/empty_context_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# CurrentAttributes without any values set result in nil attributes during job execution + +setup_localstack +setup_active_job + +require 'active_support/current_attributes' +require 'shoryuken/active_job/current_attributes' + +queue_name = DT.queue +create_test_queue(queue_name) + +class TestCurrent < ActiveSupport::CurrentAttributes + attribute :user_id, :tenant_id +end + +Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent) + +class EmptyContextTestJob < ActiveJob::Base + def perform + DT[:executions] << { + user_id: TestCurrent.user_id, + tenant_id: TestCurrent.tenant_id + } + end +end + +EmptyContextTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +EmptyContextTestJob.perform_later + +poll_queues_until(timeout: 30) { DT[:executions].size >= 1 } + +result = DT[:executions].first +assert(result[:user_id].nil?) +assert(result[:tenant_id].nil?) diff --git a/spec/integration/active_job/current_attributes/full_context_spec.rb b/spec/integration/active_job/current_attributes/full_context_spec.rb new file mode 100644 index 00000000..88c696f7 --- /dev/null +++ b/spec/integration/active_job/current_attributes/full_context_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +# CurrentAttributes with full context are persisted and restored during job execution + +setup_localstack +setup_active_job + +require 'active_support/current_attributes' +require 'shoryuken/active_job/current_attributes' + +queue_name = DT.queue +create_test_queue(queue_name) + +class TestCurrent < ActiveSupport::CurrentAttributes + attribute :user_id, :tenant_id, :request_id +end + +class RequestContext < ActiveSupport::CurrentAttributes + attribute :locale, :timezone, :trace_id +end + +Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent, RequestContext) + +class FullContextTestJob < ActiveJob::Base + def perform + DT[:executions] << { + user_id: TestCurrent.user_id, + tenant_id: TestCurrent.tenant_id, + request_id: TestCurrent.request_id, + locale: RequestContext.locale, + timezone: RequestContext.timezone, + trace_id: RequestContext.trace_id + } + end +end + +FullContextTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +TestCurrent.user_id = 42 +TestCurrent.tenant_id = 'acme-corp' +TestCurrent.request_id = 'req-123-abc' +RequestContext.locale = 'en-US' +RequestContext.timezone = 'America/New_York' +RequestContext.trace_id = 'trace-xyz-789' + +FullContextTestJob.perform_later + +TestCurrent.reset +RequestContext.reset + +poll_queues_until(timeout: 30) { DT[:executions].size >= 1 } + +result = DT[:executions].first +assert_equal(42, result[:user_id]) +assert_equal('acme-corp', result[:tenant_id]) +assert_equal('req-123-abc', result[:request_id]) +assert_equal('en-US', result[:locale]) +assert_equal('America/New_York', result[:timezone]) +assert_equal('trace-xyz-789', result[:trace_id]) diff --git a/spec/integration/active_job/current_attributes/partial_context_spec.rb b/spec/integration/active_job/current_attributes/partial_context_spec.rb new file mode 100644 index 00000000..8fba73f1 --- /dev/null +++ b/spec/integration/active_job/current_attributes/partial_context_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +# CurrentAttributes with partial values set preserve only set attributes during job execution + +setup_localstack +setup_active_job + +require 'active_support/current_attributes' +require 'shoryuken/active_job/current_attributes' + +queue_name = DT.queue +create_test_queue(queue_name) + +class TestCurrent < ActiveSupport::CurrentAttributes + attribute :user_id, :tenant_id, :request_id +end + +class RequestContext < ActiveSupport::CurrentAttributes + attribute :locale, :timezone +end + +Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent, RequestContext) + +class PartialContextTestJob < ActiveJob::Base + def perform + DT[:executions] << { + user_id: TestCurrent.user_id, + tenant_id: TestCurrent.tenant_id, + request_id: TestCurrent.request_id, + locale: RequestContext.locale, + timezone: RequestContext.timezone + } + end +end + +PartialContextTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +TestCurrent.user_id = 99 +RequestContext.locale = 'fr-FR' + +PartialContextTestJob.perform_later + +TestCurrent.reset +RequestContext.reset + +poll_queues_until(timeout: 30) { DT[:executions].size >= 1 } + +result = DT[:executions].first +assert_equal(99, result[:user_id]) +assert(result[:tenant_id].nil?) +assert(result[:request_id].nil?) +assert_equal('fr-FR', result[:locale]) +assert(result[:timezone].nil?) diff --git a/spec/integration/active_job/custom_attributes/number_attributes_spec.rb b/spec/integration/active_job/custom_attributes/number_attributes_spec.rb new file mode 100644 index 00000000..18b423a9 --- /dev/null +++ b/spec/integration/active_job/custom_attributes/number_attributes_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# ActiveJob custom numeric message attributes are sent to SQS with correct data type + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name) + +class NumberAttributesTestJob < ActiveJob::Base + def perform + DT[:executions] << { job_id: job_id } + end +end + +NumberAttributesTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +job = NumberAttributesTestJob.new +job.sqs_send_message_parameters = { + message_attributes: { + 'priority' => { string_value: '10', data_type: 'Number' }, + 'retry_count' => { string_value: '0', data_type: 'Number' } + } +} +ActiveJob::QueueAdapters::ShoryukenAdapter.enqueue(job) + +poll_queues_until(timeout: 30) { DT[:executions].size >= 1 } + +params = job.sqs_send_message_parameters +assert(params[:message_attributes].key?('priority')) +assert_equal('10', params[:message_attributes]['priority'][:string_value]) +assert_equal('Number', params[:message_attributes]['priority'][:data_type]) diff --git a/spec/integration/active_job/custom_attributes/string_attributes_spec.rb b/spec/integration/active_job/custom_attributes/string_attributes_spec.rb new file mode 100644 index 00000000..d42a60be --- /dev/null +++ b/spec/integration/active_job/custom_attributes/string_attributes_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# ActiveJob custom string message attributes are sent to SQS and preserved + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name) + +class StringAttributesTestJob < ActiveJob::Base + def perform + DT[:executions] << { job_id: job_id } + end +end + +StringAttributesTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +job = StringAttributesTestJob.new +job.sqs_send_message_parameters = { + message_attributes: { + 'trace_id' => { string_value: 'trace-abc-123', data_type: 'String' }, + 'correlation_id' => { string_value: 'corr-xyz-789', data_type: 'String' } + } +} +ActiveJob::QueueAdapters::ShoryukenAdapter.enqueue(job) + +poll_queues_until(timeout: 30) { DT[:executions].size >= 1 } + +params = job.sqs_send_message_parameters +assert(params[:message_attributes].key?('trace_id')) +assert(params[:message_attributes].key?('correlation_id')) +assert(params[:message_attributes].key?('shoryuken_class')) +assert_equal('trace-abc-123', params[:message_attributes]['trace_id'][:string_value]) +assert_equal('corr-xyz-789', params[:message_attributes]['correlation_id'][:string_value]) diff --git a/spec/integration/active_job/error_handling/job_wrapper_spec.rb b/spec/integration/active_job/error_handling/job_wrapper_spec.rb new file mode 100644 index 00000000..abb98e43 --- /dev/null +++ b/spec/integration/active_job/error_handling/job_wrapper_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# This spec tests error handling including retry configuration, +# discard configuration, and job processing through JobWrapper. + +setup_active_job + +class RetryableJob < ActiveJob::Base + queue_as :default + retry_on StandardError, wait: 1.second, attempts: 3 + + def perform(should_fail = true) + raise StandardError, 'Job failed!' if should_fail + 'Job succeeded!' + end +end + +class DiscardableJob < ActiveJob::Base + queue_as :default + discard_on ArgumentError + + def perform(should_fail = false) + raise ArgumentError, 'Invalid argument' if should_fail + 'Job succeeded!' + end +end + +job_capture = JobCapture.new +job_capture.start_capturing + +RetryableJob.perform_later(false) + +assert_equal(1, job_capture.job_count) +job = job_capture.last_job +message_body = job[:message_body] +assert_equal('RetryableJob', message_body['job_class']) +assert_equal([false], message_body['arguments']) + +job_capture2 = JobCapture.new +job_capture2.start_capturing + +DiscardableJob.perform_later(false) + +assert_equal(1, job_capture2.job_count) +job2 = job_capture2.last_job +message_body2 = job2[:message_body] +assert_equal('DiscardableJob', message_body2['job_class']) + +wrapper_class = Shoryuken::ActiveJob::JobWrapper +options = wrapper_class.get_shoryuken_options + +assert_equal(:json, options['body_parser']) +assert_equal(true, options['auto_delete']) diff --git a/spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb b/spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb new file mode 100644 index 00000000..ab5db999 --- /dev/null +++ b/spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'digest' + +# This spec tests FIFO queue support including message deduplication ID generation +# and message attributes handling. + +setup_active_job + +class FifoTestJob < ActiveJob::Base + queue_as :test_fifo + + def perform(order_id, action) + "Processed order #{order_id}: #{action}" + end +end + +class AttributesTestJob < ActiveJob::Base + queue_as :attributes_test + + def perform(data) + "Processed: #{data}" + end +end + +fifo_queue_mock = Object.new +fifo_queue_mock.define_singleton_method(:fifo?) { true } +fifo_queue_mock.define_singleton_method(:name) { 'test_fifo.fifo' } + +captured_params = nil +fifo_queue_mock.define_singleton_method(:send_message) do |params| + captured_params = params +end + +Shoryuken::Client.define_singleton_method(:queues) do |queue_name = nil| + if queue_name + fifo_queue_mock + else + { test_fifo: fifo_queue_mock } + end +end + +Shoryuken.define_singleton_method(:register_worker) { |*args| nil } + +FifoTestJob.perform_later('order-123', 'process') + +assert(captured_params.key?(:message_deduplication_id)) +assert_equal(64, captured_params[:message_deduplication_id].length) + +body = captured_params[:message_body] +body_without_variable_fields = body.except('job_id', 'enqueued_at') +expected_dedupe_id = Digest::SHA256.hexdigest(JSON.dump(body_without_variable_fields)) +assert_equal(expected_dedupe_id, captured_params[:message_deduplication_id]) + +regular_queue_mock = Object.new +regular_queue_mock.define_singleton_method(:fifo?) { false } +regular_queue_mock.define_singleton_method(:name) { 'attributes_test' } + +captured_attrs = nil +regular_queue_mock.define_singleton_method(:send_message) do |params| + captured_attrs = params +end + +Shoryuken::Client.define_singleton_method(:queues) do |queue_name = nil| + regular_queue_mock +end + +custom_attributes = { + 'trace_id' => { string_value: 'trace-123', data_type: 'String' }, + 'priority' => { string_value: 'high', data_type: 'String' } +} + +job = AttributesTestJob.new('test data') +job.sqs_send_message_parameters = { message_attributes: custom_attributes } +ActiveJob::QueueAdapters::ShoryukenAdapter.enqueue(job) + +attributes = captured_attrs[:message_attributes] +assert_equal(custom_attributes['trace_id'], attributes['trace_id']) +assert_equal(custom_attributes['priority'], attributes['priority']) + +# Should still include required Shoryuken attribute +expected_shoryuken_class = { + string_value: "Shoryuken::ActiveJob::JobWrapper", + data_type: 'String' +} +assert_equal(expected_shoryuken_class, attributes['shoryuken_class']) diff --git a/spec/integration/active_job/retry/discard_on_spec.rb b/spec/integration/active_job/retry/discard_on_spec.rb new file mode 100644 index 00000000..44a566a0 --- /dev/null +++ b/spec/integration/active_job/retry/discard_on_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# ActiveJob discard_on discards jobs that raise specific errors without retry + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name) + +class DiscardOnTestJob < ActiveJob::Base + discard_on ArgumentError + + def perform(should_fail) + DT[:attempts] << { job_id: job_id, should_fail: should_fail } + + if should_fail + raise ArgumentError, "This should be discarded" + end + + DT[:successes] << { job_id: job_id } + end +end + +DiscardOnTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +failing_job = DiscardOnTestJob.perform_later(true) +success_job = DiscardOnTestJob.perform_later(false) + +poll_queues_until(timeout: 30) { DT[:attempts].size >= 2 } + +failing_attempts = DT[:attempts].select { |a| a[:job_id] == failing_job.job_id } +assert_equal(1, failing_attempts.size, "Discarded job should only attempt once") + +failing_successes = DT[:successes].select { |s| s[:job_id] == failing_job.job_id } +assert_equal(0, failing_successes.size, "Discarded job should not succeed") + +success_successes = DT[:successes].select { |s| s[:job_id] == success_job.job_id } +assert_equal(1, success_successes.size, "Non-failing job should succeed") diff --git a/spec/integration/active_job/retry/retry_on_spec.rb b/spec/integration/active_job/retry/retry_on_spec.rb new file mode 100644 index 00000000..3a0d1d42 --- /dev/null +++ b/spec/integration/active_job/retry/retry_on_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# ActiveJob retry_on re-enqueues failed jobs until they succeed or exhaust attempts + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' }) + +class RetryOnTestJob < ActiveJob::Base + retry_on StandardError, wait: 0, attempts: 3 + + def perform + DT[:attempts] << { job_id: job_id, attempt: executions + 1, time: Time.now } + + if DT[:attempts].count { |a| a[:job_id] == job_id } < 3 + raise StandardError, "Simulated failure" + end + + DT[:successes] << { job_id: job_id, final_attempt: executions + 1 } + end +end + +RetryOnTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +RetryOnTestJob.perform_later + +poll_queues_until(timeout: 30) { DT[:successes].size >= 1 } + +assert(DT[:attempts].size >= 2, "Expected at least 2 retry attempts, got #{DT[:attempts].size}") +assert_equal(1, DT[:successes].size) diff --git a/spec/integration/active_job/roundtrip/roundtrip_spec.rb b/spec/integration/active_job/roundtrip/roundtrip_spec.rb new file mode 100644 index 00000000..09f2d182 --- /dev/null +++ b/spec/integration/active_job/roundtrip/roundtrip_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# Full round-trip ActiveJob integration test +# Enqueues a job via ActiveJob → sends to LocalStack SQS → processes via Shoryuken → verifies execution + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name) + +class RoundtripTestJob < ActiveJob::Base + def perform(payload) + DT[:executions] << { + payload: payload, + executed_at: Time.now, + job_id: job_id + } + end +end + +RoundtripTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +RoundtripTestJob.perform_later('first_payload') +RoundtripTestJob.perform_later('second_payload') +RoundtripTestJob.perform_later({ key: 'complex', data: [1, 2, 3] }) + +poll_queues_until(timeout: 30) do + DT[:executions].size >= 3 +end + +assert_equal(3, DT[:executions].size, "Expected 3 job executions, got #{DT[:executions].size}") + +payloads = DT[:executions].map { |e| e[:payload] } +assert_includes(payloads, 'first_payload') +assert_includes(payloads, 'second_payload') + +complex_payload = payloads.find { |p| p.is_a?(Hash) } +assert(complex_payload, "Expected to find complex payload") +# Keys may be strings or symbols depending on serialization +key_value = complex_payload['key'] || complex_payload[:key] +data_value = complex_payload['data'] || complex_payload[:data] +assert_equal('complex', key_value) +assert_equal([1, 2, 3], data_value) + +job_ids = DT[:executions].map { |e| e[:job_id] } +assert(job_ids.all? { |id| id && !id.empty? }, "All jobs should have job IDs") +assert_equal(3, job_ids.uniq.size, "All job IDs should be unique") diff --git a/spec/integration/active_job/scheduled/scheduled_spec.rb b/spec/integration/active_job/scheduled/scheduled_spec.rb new file mode 100644 index 00000000..6708209e --- /dev/null +++ b/spec/integration/active_job/scheduled/scheduled_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +# Scheduled ActiveJob integration test +# Tests jobs scheduled with set(wait:) are delivered after the delay + +setup_localstack +setup_active_job + +queue_name = DT.queue +create_test_queue(queue_name) + +class ScheduledTestJob < ActiveJob::Base + def perform(label) + DT[:executions] << { + label: label, + job_id: job_id, + executed_at: Time.now + } + end +end + +ScheduledTestJob.queue_as(queue_name) + +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper) + +immediate_enqueue_time = Time.now +ScheduledTestJob.perform_later('immediate') +DT[:timestamps] << { label: 'immediate', time: immediate_enqueue_time } + +delayed_enqueue_time = Time.now +ScheduledTestJob.set(wait: 3.seconds).perform_later('delayed_3s') +DT[:timestamps] << { label: 'delayed_3s', time: delayed_enqueue_time } + +delayed_5s_enqueue_time = Time.now +ScheduledTestJob.set(wait: 5.seconds).perform_later('delayed_5s') +DT[:timestamps] << { label: 'delayed_5s', time: delayed_5s_enqueue_time } + +poll_queues_until(timeout: 30) do + DT[:executions].size >= 3 +end + +assert_equal(3, DT[:executions].size, "Expected 3 job executions") + +# Find each job's execution +immediate_job = DT[:executions].find { |e| e[:label] == 'immediate' } +delayed_3s_job = DT[:executions].find { |e| e[:label] == 'delayed_3s' } +delayed_5s_job = DT[:executions].find { |e| e[:label] == 'delayed_5s' } + +assert(immediate_job, "Immediate job should have executed") +assert(delayed_3s_job, "3s delayed job should have executed") +assert(delayed_5s_job, "5s delayed job should have executed") + +def enqueue_time(label) + DT[:timestamps].find { |t| t[:label] == label }[:time] +end + +immediate_delay = immediate_job[:executed_at] - enqueue_time('immediate') +assert(immediate_delay < 10, "Immediate job should execute within 10 seconds, took #{immediate_delay}s") + +# Using 2 seconds tolerance for SQS delivery variation +delayed_3s_actual_delay = delayed_3s_job[:executed_at] - enqueue_time('delayed_3s') +assert(delayed_3s_actual_delay >= 2, "3s delayed job should execute after at least 2s, took #{delayed_3s_actual_delay}s") + +delayed_5s_actual_delay = delayed_5s_job[:executed_at] - enqueue_time('delayed_5s') +assert(delayed_5s_actual_delay >= 4, "5s delayed job should execute after at least 4s, took #{delayed_5s_actual_delay}s") + +assert( + immediate_job[:executed_at] <= delayed_3s_job[:executed_at], + "Immediate job should execute before 3s delayed job" +) +assert( + delayed_3s_job[:executed_at] <= delayed_5s_job[:executed_at], + "3s delayed job should execute before 5s delayed job" +) diff --git a/spec/integration/active_job_continuation_spec.rb b/spec/integration/active_job_continuation_spec.rb deleted file mode 100644 index b6e32920..00000000 --- a/spec/integration/active_job_continuation_spec.rb +++ /dev/null @@ -1,145 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' -require 'active_job' -require 'shoryuken/extensions/active_job_adapter' -require 'shoryuken/extensions/active_job_extensions' - -RSpec.describe 'ActiveJob Continuations Integration' do - # Skip all tests in this suite if ActiveJob::Continuable is not available (Rails < 8.0) - before(:all) do - skip 'ActiveJob::Continuable not available (Rails < 8.0)' unless defined?(ActiveJob::Continuable) - end - - # Test job that uses ActiveJob Continuations - class ContinuableTestJob < ActiveJob::Base - include ActiveJob::Continuable if defined?(ActiveJob::Continuable) - - queue_as :default - - class_attribute :executions_log, default: [] - class_attribute :checkpoints_reached, default: [] - - def perform(max_iterations: 10) - self.class.executions_log << { execution: executions, started_at: Time.current } - - step :initialize_work do - self.class.checkpoints_reached << "initialize_work_#{executions}" - end - - step :process_items, start: cursor || 0 do - (cursor..max_iterations).each do |i| - self.class.checkpoints_reached << "processing_item_#{i}" - - # Check if we should stop (checkpoint) - checkpoint - - # Simulate some work - sleep 0.01 - - # Advance cursor - cursor.advance! - end - end - - step :finalize_work do - self.class.checkpoints_reached << 'finalize_work' - end - - self.class.executions_log.last[:completed] = true - end - end - - describe 'stopping? method (unit tests)' do - it 'returns false when launcher is not initialized' do - adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new - expect(adapter.stopping?).to be false - end - - it 'returns true when launcher is stopping' do - launcher = Shoryuken::Launcher.new - runner = Shoryuken::Runner.instance - runner.instance_variable_set(:@launcher, launcher) - - adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new - expect(adapter.stopping?).to be false - - launcher.instance_variable_set(:@stopping, true) - expect(adapter.stopping?).to be true - end - end - - describe 'timestamp handling for continuation retries' do - it 'handles past timestamps for continuation retries' do - adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new - job = ContinuableTestJob.new - job.sqs_send_message_parameters = {} - - # Mock the queue - queue = instance_double(Shoryuken::Queue, fifo?: false) - allow(Shoryuken::Client).to receive(:queues).and_return(queue) - allow(Shoryuken).to receive(:register_worker) - allow(queue).to receive(:send_message) do |params| - # Verify past timestamp results in immediate delivery (delay_seconds <= 0) - expect(params[:delay_seconds]).to be <= 0 - end - - # Enqueue with past timestamp (simulating continuation retry) - past_timestamp = Time.current.to_f - 60 - adapter.enqueue_at(job, past_timestamp) - end - end - - describe 'enqueue_at with continuation timestamps (unit tests)' do - let(:adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new } - let(:job) do - job = ContinuableTestJob.new - job.sqs_send_message_parameters = {} - job - end - let(:queue) { instance_double(Shoryuken::Queue, fifo?: false) } - - before do - allow(Shoryuken::Client).to receive(:queues).and_return(queue) - allow(Shoryuken).to receive(:register_worker) - @sent_messages = [] - allow(queue).to receive(:send_message) do |params| - @sent_messages << params - end - end - - it 'accepts past timestamps without error' do - past_timestamp = Time.current.to_f - 30 - - expect { - adapter.enqueue_at(job, past_timestamp) - }.not_to raise_error - - expect(@sent_messages.size).to eq(1) - expect(@sent_messages.first[:delay_seconds]).to be <= 0 - end - - it 'accepts current timestamp' do - current_timestamp = Time.current.to_f - - expect { - adapter.enqueue_at(job, current_timestamp) - }.not_to raise_error - - expect(@sent_messages.size).to eq(1) - expect(@sent_messages.first[:delay_seconds]).to be_between(-1, 1) - end - - it 'accepts future timestamp' do - future_timestamp = Time.current.to_f + 30 - - expect { - adapter.enqueue_at(job, future_timestamp) - }.not_to raise_error - - expect(@sent_messages.size).to eq(1) - expect(@sent_messages.first[:delay_seconds]).to be > 0 - expect(@sent_messages.first[:delay_seconds]).to be <= 30 - end - end -end diff --git a/spec/integration/batch_processing/batch_processing_spec.rb b/spec/integration/batch_processing/batch_processing_spec.rb new file mode 100644 index 00000000..79552d00 --- /dev/null +++ b/spec/integration/batch_processing/batch_processing_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# This spec tests batch processing including batch message reception (up to 10 +# messages), batch vs single worker behavior differences, JSON body parsing in +# batch mode, and maximum batch size handling. + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + def perform(sqs_msgs, bodies) + msgs = Array(sqs_msgs) + DT[:batch_sizes] << msgs.size + DT[:messages].concat(Array(bodies)) + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +worker_class.get_shoryuken_options['auto_delete'] = true +worker_class.get_shoryuken_options['batch'] = true +Shoryuken.register_worker(queue_name, worker_class) + +entries = 5.times.map { |i| { id: SecureRandom.uuid, message_body: "message-#{i}" } } +Shoryuken::Client.queues(queue_name).send_messages(entries: entries) + +sleep 1 + +poll_queues_until { DT[:messages].size >= 5 } + +assert_equal(5, DT[:messages].size) +assert(DT[:batch_sizes].any? { |size| size > 1 }, "Expected at least one batch with size > 1") diff --git a/spec/integration/concurrent_processing/concurrent_processing_spec.rb b/spec/integration/concurrent_processing/concurrent_processing_spec.rb new file mode 100644 index 00000000..835977d5 --- /dev/null +++ b/spec/integration/concurrent_processing/concurrent_processing_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# This spec tests concurrent message processing with multiple processors. + +require 'concurrent' + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name) +Shoryuken.add_group('concurrent', 5) # 5 concurrent processors +Shoryuken.add_queue(queue_name, 1, 'concurrent') + +# Atomic counters for tracking concurrency +concurrent_count = Concurrent::AtomicFixnum.new(0) +max_concurrent = Concurrent::AtomicFixnum.new(0) + +worker_class = Class.new do + include Shoryuken::Worker + + shoryuken_options auto_delete: true, batch: false + + define_method(:perform) do |sqs_msg, body| + concurrent_count.increment + current = concurrent_count.value + max_concurrent.update { |max| [max, current].max } + + sleep 0.5 # Simulate work + + DT[:processing_times] << Time.now + + concurrent_count.decrement + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +Shoryuken.register_worker(queue_name, worker_class) + +10.times { |i| Shoryuken::Client.queues(queue_name).send_message(message_body: "msg-#{i}") } + +poll_queues_until(timeout: 20) { DT[:processing_times].size >= 10 } + +assert_equal(10, DT[:processing_times].size) +# With multiple processors, we should see concurrency > 1 +assert(max_concurrent.value > 1, "Expected concurrency > 1, got #{max_concurrent.value}") diff --git a/spec/integration/fifo_ordering/fifo_ordering_spec.rb b/spec/integration/fifo_ordering/fifo_ordering_spec.rb new file mode 100644 index 00000000..cf3cee76 --- /dev/null +++ b/spec/integration/fifo_ordering/fifo_ordering_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# This spec tests FIFO queue ordering guarantees including message ordering +# within the same message group. + +setup_localstack + +queue_name = "#{DT.uuid}.fifo" +create_fifo_queue(queue_name) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + def perform(sqs_msg, body) + DT[:messages] << body + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +worker_class.get_shoryuken_options['auto_delete'] = true +worker_class.get_shoryuken_options['batch'] = false +Shoryuken.register_worker(queue_name, worker_class) + +queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url + +5.times do |i| + Shoryuken::Client.sqs.send_message( + queue_url: queue_url, + message_body: "msg-#{i}", + message_group_id: 'group-a', + message_deduplication_id: SecureRandom.uuid + ) +end + +sleep 1 + +poll_queues_until { DT[:messages].size >= 5 } + +assert_equal(5, DT[:messages].size) + +expected = (0..4).map { |i| "msg-#{i}" } +assert_equal(expected, DT[:messages]) diff --git a/spec/integration/large_payloads/large_payloads_spec.rb b/spec/integration/large_payloads/large_payloads_spec.rb new file mode 100644 index 00000000..c604924c --- /dev/null +++ b/spec/integration/large_payloads/large_payloads_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# This spec tests large payload handling including payloads near the 256KB SQS limit. + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + def perform(sqs_msg, body) + DT[:bodies] << body + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +worker_class.get_shoryuken_options['auto_delete'] = true +worker_class.get_shoryuken_options['batch'] = false +Shoryuken.register_worker(queue_name, worker_class) + +payload = 'x' * (250 * 1024) +Shoryuken::Client.queues(queue_name).send_message(message_body: payload) + +poll_queues_until { DT[:bodies].size >= 1 } + +assert_equal(250 * 1024, DT[:bodies].first.size) diff --git a/spec/integration/launcher/launcher_spec.rb b/spec/integration/launcher/launcher_spec.rb new file mode 100644 index 00000000..242b22b0 --- /dev/null +++ b/spec/integration/launcher/launcher_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# This spec tests the Launcher's ability to consume messages from SQS queues, +# including single message consumption, batch consumption, and command workers. + +require 'concurrent' + +setup_localstack + +# Use atomic counter for thread-safe message counting +message_counter = Concurrent::AtomicFixnum.new(0) + +worker_class = Class.new do + include Shoryuken::Worker + + shoryuken_options auto_delete: true + + define_method(:perform) do |sqs_msg, _body| + message_counter.increment(Array(sqs_msg).size) + end +end + +queue_name = DT.queue + +create_test_queue(queue_name) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') +worker_class.get_shoryuken_options['queue'] = queue_name +worker_class.get_shoryuken_options['batch'] = true +Shoryuken.register_worker(queue_name, worker_class) + +entries = 10.times.map { |i| { id: SecureRandom.uuid, message_body: i.to_s } } +Shoryuken::Client.queues(queue_name).send_messages(entries: entries) + +# Give the messages a chance to hit the queue +sleep 2 + +poll_queues_until { message_counter.value > 0 } + +assert(message_counter.value > 1, "Expected more than 1 message in batch, got #{message_counter.value}") diff --git a/spec/integration/launcher_spec.rb b/spec/integration/launcher_spec.rb deleted file mode 100644 index 0553cf8a..00000000 --- a/spec/integration/launcher_spec.rb +++ /dev/null @@ -1,127 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' - -RSpec.describe Shoryuken::Launcher do - let(:sqs_client) do - Aws::SQS::Client.new( - region: 'us-east-1', - endpoint: 'http://localhost:4566', - access_key_id: 'fake', - secret_access_key: 'fake' - ) - end - - let(:executor) do - # We can't use Concurrent.global_io_executor in these tests since once you - # shut down a thread pool, you can't start it back up. Instead, we create - # one new thread pool executor for each spec. We use a new - # CachedThreadPool, since that most closely resembles - # Concurrent.global_io_executor - Concurrent::CachedThreadPool.new auto_terminate: true - end - - describe 'Consuming messages' do - before do - Aws.config[:stub_responses] = false - - allow(Shoryuken).to receive(:launcher_executor).and_return(executor) - - Shoryuken.configure_client do |config| - config.sqs_client = sqs_client - end - - Shoryuken.configure_server do |config| - config.sqs_client = sqs_client - end - - StandardWorker.received_messages = 0 - - queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}" - - Shoryuken::Client.sqs.create_queue(queue_name: queue) - - Shoryuken.add_group('default', 1) - Shoryuken.add_queue(queue, 1, 'default') - - StandardWorker.get_shoryuken_options['queue'] = queue - - Shoryuken.register_worker(queue, StandardWorker) - end - - after do - Aws.config[:stub_responses] = true - - queue_url = Shoryuken::Client.sqs.get_queue_url( - queue_name: StandardWorker.get_shoryuken_options['queue'] - ).queue_url - - Shoryuken::Client.sqs.delete_queue(queue_url: queue_url) - end - - it 'consumes as a command worker' do - StandardWorker.perform_async('Yo') - - poll_queues_until { StandardWorker.received_messages > 0 } - - expect(StandardWorker.received_messages).to eq 1 - end - - it 'consumes a message' do - StandardWorker.get_shoryuken_options['batch'] = false - - Shoryuken::Client.queues(StandardWorker.get_shoryuken_options['queue']).send_message(message_body: 'Yo') - - poll_queues_until { StandardWorker.received_messages > 0 } - - expect(StandardWorker.received_messages).to eq 1 - end - - it 'consumes a batch' do - StandardWorker.get_shoryuken_options['batch'] = true - - entries = 10.times.map { |i| { id: SecureRandom.uuid, message_body: i.to_s } } - - Shoryuken::Client.queues(StandardWorker.get_shoryuken_options['queue']).send_messages(entries: entries) - - # Give the messages a chance to hit the queue so they are all available at the same time - sleep 2 - - poll_queues_until { StandardWorker.received_messages > 0 } - - expect(StandardWorker.received_messages).to be > 1 - end - - def poll_queues_until - subject.start - - Timeout::timeout(10) do - begin - sleep 0.5 - end until yield - end - ensure - subject.stop - end - - class StandardWorker - include Shoryuken::Worker - - @@received_messages = 0 - - shoryuken_options auto_delete: true - - def perform(sqs_msg, _body) - @@received_messages += Array(sqs_msg).size - end - - def self.received_messages - @@received_messages - end - - def self.received_messages=(received_messages) - @@received_messages = received_messages - end - end - end -end diff --git a/spec/integration/message_attributes/message_attributes_spec.rb b/spec/integration/message_attributes/message_attributes_spec.rb new file mode 100644 index 00000000..b2106693 --- /dev/null +++ b/spec/integration/message_attributes/message_attributes_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This spec tests SQS message attributes including String, Number, and Binary +# attribute types, system attributes (ApproximateReceiveCount, SentTimestamp), +# and custom type suffixes. + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + shoryuken_options auto_delete: true, batch: false + + def perform(sqs_msg, body) + DT[:attributes] << sqs_msg.message_attributes + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +Shoryuken.register_worker(queue_name, worker_class) + +queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url + +Shoryuken::Client.sqs.send_message( + queue_url: queue_url, + message_body: 'mixed-attr-test', + message_attributes: { + 'StringAttr' => { + string_value: 'hello-world', + data_type: 'String' + }, + 'NumberAttr' => { + string_value: '42', + data_type: 'Number' + }, + 'BinaryAttr' => { + binary_value: 'binary-data'.b, + data_type: 'Binary' + } + } +) + +poll_queues_until { DT[:attributes].size >= 1 } + +attrs = DT[:attributes].first +assert_equal(3, attrs.keys.size) +assert_equal('hello-world', attrs['StringAttr']&.string_value) +assert_equal('42', attrs['NumberAttr']&.string_value) +assert_equal('binary-data'.b, attrs['BinaryAttr']&.binary_value) diff --git a/spec/integration/middleware_chain/empty_chain_spec.rb b/spec/integration/middleware_chain/empty_chain_spec.rb new file mode 100644 index 00000000..2603ad54 --- /dev/null +++ b/spec/integration/middleware_chain/empty_chain_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# Empty middleware chain executes worker block directly + +chain = Shoryuken::Middleware::Chain.new + +chain.invoke(nil, 'test', nil, nil) do + DT[:calls] << :worker +end + +assert_equal([:worker], DT[:calls]) diff --git a/spec/integration/middleware_chain/execution_order_spec.rb b/spec/integration/middleware_chain/execution_order_spec.rb new file mode 100644 index 00000000..b597516d --- /dev/null +++ b/spec/integration/middleware_chain/execution_order_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# Middleware executes in onion model order (first-in wraps outermost) + +def create_middleware(name) + Class.new do + define_method(:call) do |worker, queue, sqs_msg, body, &block| + DT[:order] << :"#{name}_before" + block.call + DT[:order] << :"#{name}_after" + end + end +end + +first = create_middleware(:first) +second = create_middleware(:second) +third = create_middleware(:third) + +chain = Shoryuken::Middleware::Chain.new +chain.add first +chain.add second +chain.add third + +chain.invoke(nil, 'test-queue', nil, nil) do + DT[:order] << :worker_perform +end + +expected_order = [ + :first_before, :second_before, :third_before, + :worker_perform, + :third_after, :second_after, :first_after +] +assert_equal(expected_order, DT[:order]) diff --git a/spec/integration/middleware_chain/removal_spec.rb b/spec/integration/middleware_chain/removal_spec.rb new file mode 100644 index 00000000..0662757a --- /dev/null +++ b/spec/integration/middleware_chain/removal_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# Middleware can be removed from the chain + +def create_middleware(name) + Class.new do + define_method(:call) do |worker, queue, sqs_msg, body, &block| + DT[:calls] << :"#{name}_before" + block.call + DT[:calls] << :"#{name}_after" + end + end +end + +first = create_middleware(:first) +second = create_middleware(:second) +third = create_middleware(:third) + +chain = Shoryuken::Middleware::Chain.new +chain.add first +chain.add second +chain.add third +chain.remove second + +chain.invoke(nil, 'test', nil, nil) do + DT[:calls] << :worker +end + +assert_includes(DT[:calls], :first_before) +refute(DT[:calls].include?(:second_before), "Second should be removed") +assert_includes(DT[:calls], :third_before) diff --git a/spec/integration/middleware_chain/short_circuit_spec.rb b/spec/integration/middleware_chain/short_circuit_spec.rb new file mode 100644 index 00000000..5a8d88d1 --- /dev/null +++ b/spec/integration/middleware_chain/short_circuit_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Middleware can short-circuit the chain by not calling the block + +def create_middleware(name) + Class.new do + define_method(:call) do |worker, queue, sqs_msg, body, &block| + DT[:calls] << :"#{name}_before" + block.call + DT[:calls] << :"#{name}_after" + end + end +end + +def create_short_circuit_middleware + Class.new do + define_method(:call) do |worker, queue, sqs_msg, body, &block| + DT[:calls] << :short_circuit + end + end +end + +first = create_middleware(:first) +short_circuit = create_short_circuit_middleware +third = create_middleware(:third) + +chain = Shoryuken::Middleware::Chain.new +chain.add first +chain.add short_circuit +chain.add third + +chain.invoke(nil, 'test', nil, nil) do + DT[:calls] << :worker +end + +assert_includes(DT[:calls], :first_before) +assert_includes(DT[:calls], :short_circuit) +refute(DT[:calls].include?(:third_before), "Third should not execute") +refute(DT[:calls].include?(:worker), "Worker should not execute") +assert_includes(DT[:calls], :first_after) diff --git a/spec/integration/polling_strategies/polling_strategies_spec.rb b/spec/integration/polling_strategies/polling_strategies_spec.rb new file mode 100644 index 00000000..71f4b034 --- /dev/null +++ b/spec/integration/polling_strategies/polling_strategies_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# This spec tests polling strategies including WeightedRoundRobin (default) +# with multi-queue worker message distribution. + +setup_localstack + +queue_high = DT.queues[0] +queue_medium = DT.queues[1] +queue_low = DT.queues[2] + +[queue_high, queue_medium, queue_low].each { |q| create_test_queue(q) } + +Shoryuken.add_group('default', 1) +# Higher weight = higher priority +Shoryuken.add_queue(queue_high, 3, 'default') +Shoryuken.add_queue(queue_medium, 2, 'default') +Shoryuken.add_queue(queue_low, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + shoryuken_options auto_delete: true, batch: false + + def perform(sqs_msg, body) + queue = sqs_msg.queue_url.split('/').last + DT[:by_queue] << { queue: queue, body: body } + end +end + +[queue_high, queue_medium, queue_low].each do |queue| + worker_class.get_shoryuken_options['queue'] = queue + Shoryuken.register_worker(queue, worker_class) +end + +Shoryuken::Client.queues(queue_high).send_message(message_body: 'high-msg') +Shoryuken::Client.queues(queue_medium).send_message(message_body: 'medium-msg') +Shoryuken::Client.queues(queue_low).send_message(message_body: 'low-msg') + +sleep 1 + +poll_queues_until { DT[:by_queue].size >= 3 } + +queues_with_messages = DT[:by_queue].map { |m| m[:queue] }.uniq +assert_equal(3, queues_with_messages.size) +assert_equal(3, DT[:by_queue].size) diff --git a/spec/integration/rails/rails_72/Gemfile b/spec/integration/rails/rails_72/Gemfile new file mode 100644 index 00000000..c8e7e848 --- /dev/null +++ b/spec/integration/rails/rails_72/Gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' + +gemspec path: '../../../../' + +gem 'activejob', '~> 7.2' +gem 'rails', '~> 7.2' diff --git a/spec/integration/rails/rails_72/activejob_adapter_spec.rb b/spec/integration/rails/rails_72/activejob_adapter_spec.rb new file mode 100644 index 00000000..9dacc886 --- /dev/null +++ b/spec/integration/rails/rails_72/activejob_adapter_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# ActiveJob adapter integration tests for Rails 7.2 + +setup_active_job + +class EmailJob < ActiveJob::Base + queue_as :default + + def perform(user_id, message) + { user_id: user_id, message: message, sent_at: Time.current } + end +end + +class DataProcessingJob < ActiveJob::Base + queue_as :high_priority + + def perform(data_file) + "Processed: #{data_file}" + end +end + +class SerializationJob < ActiveJob::Base + queue_as :default + + def perform(complex_data) + complex_data.transform_values(&:upcase) + end +end + +# Test adapter setup +adapter = ActiveJob::Base.queue_adapter +assert_equal("ActiveJob::QueueAdapters::ShoryukenAdapter", adapter.class.name) + +# Test singleton pattern +instance1 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +instance2 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +assert_equal(instance1.object_id, instance2.object_id) + +# Test transaction commit hook (Rails 7.2+) +adapter_instance = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert(adapter_instance.respond_to?(:enqueue_after_transaction_commit?)) +assert_equal(true, adapter_instance.enqueue_after_transaction_commit?) + +# Test simple job enqueue +job_capture = JobCapture.new +job_capture.start_capturing + +EmailJob.perform_later(1, 'Hello World') + +assert_equal(1, job_capture.job_count) +job = job_capture.last_job +message_body = job[:message_body] +assert_equal('EmailJob', message_body['job_class']) +assert_equal([1, 'Hello World'], message_body['arguments']) +assert_equal('default', message_body['queue_name']) + +# Test different queue +job_capture2 = JobCapture.new +job_capture2.start_capturing + +DataProcessingJob.perform_later('large_dataset.csv') + +job2 = job_capture2.last_job +message_body2 = job2[:message_body] +assert_equal('DataProcessingJob', message_body2['job_class']) +assert_equal('high_priority', message_body2['queue_name']) + +# Test complex data serialization +complex_data = { + 'user' => { 'name' => 'John', 'age' => 30 }, + 'preferences' => ['email', 'sms'] +} + +job_capture3 = JobCapture.new +job_capture3.start_capturing + +SerializationJob.perform_later(complex_data) + +job3 = job_capture3.last_job +message_body3 = job3[:message_body] +args_data = message_body3['arguments'].first +assert_equal('John', args_data['user']['name']) +assert_equal(30, args_data['user']['age']) + +# Test shoryuken_class message attribute +job_capture4 = JobCapture.new +job_capture4.start_capturing + +EmailJob.perform_later(1, 'Attributes test') + +job4 = job_capture4.last_job +attributes = job4[:message_attributes] +expected_shoryuken_class = { + string_value: "Shoryuken::ActiveJob::JobWrapper", + data_type: 'String' +} +assert_equal(expected_shoryuken_class, attributes['shoryuken_class']) diff --git a/spec/integration/rails/rails_80/Gemfile b/spec/integration/rails/rails_80/Gemfile new file mode 100644 index 00000000..e0d56349 --- /dev/null +++ b/spec/integration/rails/rails_80/Gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' + +gemspec path: '../../../../' + +gem 'activejob', '~> 8.0' +gem 'rails', '~> 8.0' diff --git a/spec/integration/rails/rails_80/activejob_adapter_spec.rb b/spec/integration/rails/rails_80/activejob_adapter_spec.rb new file mode 100644 index 00000000..8e585356 --- /dev/null +++ b/spec/integration/rails/rails_80/activejob_adapter_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# ActiveJob adapter integration tests for Rails 8.0 + +setup_active_job + +class EmailJob < ActiveJob::Base + queue_as :default + + def perform(user_id, message) + { user_id: user_id, message: message, sent_at: Time.current } + end +end + +class DataProcessingJob < ActiveJob::Base + queue_as :high_priority + + def perform(data_file) + "Processed: #{data_file}" + end +end + +class SerializationJob < ActiveJob::Base + queue_as :default + + def perform(complex_data) + complex_data.transform_values(&:upcase) + end +end + +# Test adapter setup +adapter = ActiveJob::Base.queue_adapter +assert_equal("ActiveJob::QueueAdapters::ShoryukenAdapter", adapter.class.name) + +# Test singleton pattern +instance1 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +instance2 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +assert_equal(instance1.object_id, instance2.object_id) + +# Test transaction commit hook +adapter_instance = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert(adapter_instance.respond_to?(:enqueue_after_transaction_commit?)) +assert_equal(true, adapter_instance.enqueue_after_transaction_commit?) + +# Test simple job enqueue +job_capture = JobCapture.new +job_capture.start_capturing + +EmailJob.perform_later(1, 'Hello World') + +assert_equal(1, job_capture.job_count) +job = job_capture.last_job +message_body = job[:message_body] +assert_equal('EmailJob', message_body['job_class']) +assert_equal([1, 'Hello World'], message_body['arguments']) +assert_equal('default', message_body['queue_name']) + +# Test different queue +job_capture2 = JobCapture.new +job_capture2.start_capturing + +DataProcessingJob.perform_later('large_dataset.csv') + +job2 = job_capture2.last_job +message_body2 = job2[:message_body] +assert_equal('DataProcessingJob', message_body2['job_class']) +assert_equal('high_priority', message_body2['queue_name']) + +# Test complex data serialization +complex_data = { + 'user' => { 'name' => 'John', 'age' => 30 }, + 'preferences' => ['email', 'sms'] +} + +job_capture3 = JobCapture.new +job_capture3.start_capturing + +SerializationJob.perform_later(complex_data) + +job3 = job_capture3.last_job +message_body3 = job3[:message_body] +args_data = message_body3['arguments'].first +assert_equal('John', args_data['user']['name']) +assert_equal(30, args_data['user']['age']) + +# Test shoryuken_class message attribute +job_capture4 = JobCapture.new +job_capture4.start_capturing + +EmailJob.perform_later(1, 'Attributes test') + +job4 = job_capture4.last_job +attributes = job4[:message_attributes] +expected_shoryuken_class = { + string_value: "Shoryuken::ActiveJob::JobWrapper", + data_type: 'String' +} +assert_equal(expected_shoryuken_class, attributes['shoryuken_class']) diff --git a/spec/integration/rails/rails_80/continuation_spec.rb b/spec/integration/rails/rails_80/continuation_spec.rb new file mode 100644 index 00000000..2fd21f72 --- /dev/null +++ b/spec/integration/rails/rails_80/continuation_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# ActiveJob Continuations integration tests for Rails 8.0+ +# Tests the stopping? method and continuation timestamp handling + +setup_active_job + +# Skip if ActiveJob::Continuable is not available (Rails < 8.0) +unless defined?(ActiveJob::Continuable) + puts "Skipping continuation tests - ActiveJob::Continuable not available (requires Rails 8.0+)" + exit 0 +end + +# Test stopping? returns false when launcher is not initialized +adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert_equal(false, adapter.stopping?) + +# Test stopping? returns true when launcher is stopping +launcher = Shoryuken::Launcher.new +runner = Shoryuken::Runner.instance +runner.instance_variable_set(:@launcher, launcher) + +adapter2 = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert_equal(false, adapter2.stopping?) + +launcher.instance_variable_set(:@stopping, true) +assert_equal(true, adapter2.stopping?) + +# Reset launcher state +launcher.instance_variable_set(:@stopping, false) + +# Test past timestamps for continuation retries +job_capture = JobCapture.new +job_capture.start_capturing + +class ContinuableTestJob < ActiveJob::Base + include ActiveJob::Continuable if defined?(ActiveJob::Continuable) + queue_as :default + def perform; end +end + +adapter3 = ActiveJob::QueueAdapters::ShoryukenAdapter.new +job = ContinuableTestJob.new +job.sqs_send_message_parameters = {} + +past_timestamp = Time.current.to_f - 60 +adapter3.enqueue_at(job, past_timestamp) + +captured_job = job_capture.last_job +assert(captured_job[:delay_seconds] <= 0, "Past timestamp should result in immediate delivery") + +# Test current timestamp +job_capture2 = JobCapture.new +job_capture2.start_capturing + +job2 = ContinuableTestJob.new +job2.sqs_send_message_parameters = {} + +current_timestamp = Time.current.to_f +adapter3.enqueue_at(job2, current_timestamp) + +captured_job2 = job_capture2.last_job +delay = captured_job2[:delay_seconds] +assert(delay >= -1 && delay <= 1, "Current timestamp should have minimal delay") + +# Test future timestamp +job_capture3 = JobCapture.new +job_capture3.start_capturing + +job3 = ContinuableTestJob.new +job3.sqs_send_message_parameters = {} + +future_timestamp = Time.current.to_f + 30 +adapter3.enqueue_at(job3, future_timestamp) + +captured_job3 = job_capture3.last_job +delay3 = captured_job3[:delay_seconds] +assert(delay3 > 0, "Future timestamp should have positive delay") +assert(delay3 <= 30, "Delay should not exceed scheduled time") diff --git a/spec/integration/rails/rails_81/Gemfile b/spec/integration/rails/rails_81/Gemfile new file mode 100644 index 00000000..f01384c1 --- /dev/null +++ b/spec/integration/rails/rails_81/Gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' + +gemspec path: '../../../../' + +gem 'activejob', '~> 8.1' +gem 'rails', '~> 8.1' diff --git a/spec/integration/rails/rails_81/activejob_adapter_spec.rb b/spec/integration/rails/rails_81/activejob_adapter_spec.rb new file mode 100644 index 00000000..b9e00760 --- /dev/null +++ b/spec/integration/rails/rails_81/activejob_adapter_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# ActiveJob adapter integration tests for Rails 8.1 + +setup_active_job + +class EmailJob < ActiveJob::Base + queue_as :default + + def perform(user_id, message) + { user_id: user_id, message: message, sent_at: Time.current } + end +end + +class DataProcessingJob < ActiveJob::Base + queue_as :high_priority + + def perform(data_file) + "Processed: #{data_file}" + end +end + +class SerializationJob < ActiveJob::Base + queue_as :default + + def perform(complex_data) + complex_data.transform_values(&:upcase) + end +end + +# Test adapter setup +adapter = ActiveJob::Base.queue_adapter +assert_equal("ActiveJob::QueueAdapters::ShoryukenAdapter", adapter.class.name) + +# Test singleton pattern +instance1 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +instance2 = ActiveJob::QueueAdapters::ShoryukenAdapter.instance +assert_equal(instance1.object_id, instance2.object_id) + +# Test transaction commit hook +adapter_instance = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert(adapter_instance.respond_to?(:enqueue_after_transaction_commit?)) +assert_equal(true, adapter_instance.enqueue_after_transaction_commit?) + +# Test simple job enqueue +job_capture = JobCapture.new +job_capture.start_capturing + +EmailJob.perform_later(1, 'Hello World') + +assert_equal(1, job_capture.job_count) +job = job_capture.last_job +message_body = job[:message_body] +assert_equal('EmailJob', message_body['job_class']) +assert_equal([1, 'Hello World'], message_body['arguments']) +assert_equal('default', message_body['queue_name']) + +# Test different queue +job_capture2 = JobCapture.new +job_capture2.start_capturing + +DataProcessingJob.perform_later('large_dataset.csv') + +job2 = job_capture2.last_job +message_body2 = job2[:message_body] +assert_equal('DataProcessingJob', message_body2['job_class']) +assert_equal('high_priority', message_body2['queue_name']) + +# Test complex data serialization +complex_data = { + 'user' => { 'name' => 'John', 'age' => 30 }, + 'preferences' => ['email', 'sms'] +} + +job_capture3 = JobCapture.new +job_capture3.start_capturing + +SerializationJob.perform_later(complex_data) + +job3 = job_capture3.last_job +message_body3 = job3[:message_body] +args_data = message_body3['arguments'].first +assert_equal('John', args_data['user']['name']) +assert_equal(30, args_data['user']['age']) + +# Test shoryuken_class message attribute +job_capture4 = JobCapture.new +job_capture4.start_capturing + +EmailJob.perform_later(1, 'Attributes test') + +job4 = job_capture4.last_job +attributes = job4[:message_attributes] +expected_shoryuken_class = { + string_value: "Shoryuken::ActiveJob::JobWrapper", + data_type: 'String' +} +assert_equal(expected_shoryuken_class, attributes['shoryuken_class']) diff --git a/spec/integration/rails/rails_81/continuation_spec.rb b/spec/integration/rails/rails_81/continuation_spec.rb new file mode 100644 index 00000000..94cde58e --- /dev/null +++ b/spec/integration/rails/rails_81/continuation_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# ActiveJob Continuations integration tests for Rails 8.1+ +# Tests the stopping? method and continuation timestamp handling + +setup_active_job + +# Skip if ActiveJob::Continuable is not available (Rails < 8.1) +unless defined?(ActiveJob::Continuable) + puts "Skipping continuation tests - ActiveJob::Continuable not available (requires Rails 8.1+)" + exit 0 +end + +# Test stopping? returns false when launcher is not initialized +adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert_equal(false, adapter.stopping?) + +# Test stopping? returns true when launcher is stopping +launcher = Shoryuken::Launcher.new +runner = Shoryuken::Runner.instance +runner.instance_variable_set(:@launcher, launcher) + +adapter2 = ActiveJob::QueueAdapters::ShoryukenAdapter.new +assert_equal(false, adapter2.stopping?) + +launcher.instance_variable_set(:@stopping, true) +assert_equal(true, adapter2.stopping?) + +# Reset launcher state +launcher.instance_variable_set(:@stopping, false) + +# Test past timestamps for continuation retries +job_capture = JobCapture.new +job_capture.start_capturing + +class ContinuableTestJob < ActiveJob::Base + include ActiveJob::Continuable if defined?(ActiveJob::Continuable) + queue_as :default + def perform; end +end + +adapter3 = ActiveJob::QueueAdapters::ShoryukenAdapter.new +job = ContinuableTestJob.new +job.sqs_send_message_parameters = {} + +past_timestamp = Time.current.to_f - 60 +adapter3.enqueue_at(job, past_timestamp) + +captured_job = job_capture.last_job +assert(captured_job[:delay_seconds] <= 0, "Past timestamp should result in immediate delivery") + +# Test current timestamp +job_capture2 = JobCapture.new +job_capture2.start_capturing + +job2 = ContinuableTestJob.new +job2.sqs_send_message_parameters = {} + +current_timestamp = Time.current.to_f +adapter3.enqueue_at(job2, current_timestamp) + +captured_job2 = job_capture2.last_job +delay = captured_job2[:delay_seconds] +assert(delay >= -1 && delay <= 1, "Current timestamp should have minimal delay") + +# Test future timestamp +job_capture3 = JobCapture.new +job_capture3.start_capturing + +job3 = ContinuableTestJob.new +job3.sqs_send_message_parameters = {} + +future_timestamp = Time.current.to_f + 30 +adapter3.enqueue_at(job3, future_timestamp) + +captured_job3 = job_capture3.last_job +delay3 = captured_job3[:delay_seconds] +assert(delay3 > 0, "Future timestamp should have positive delay") +assert(delay3 <= 30, "Delay should not exceed scheduled time") diff --git a/spec/integration/retry_behavior/retry_behavior_spec.rb b/spec/integration/retry_behavior/retry_behavior_spec.rb new file mode 100644 index 00000000..a7b14226 --- /dev/null +++ b/spec/integration/retry_behavior/retry_behavior_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# This spec tests retry behavior including ApproximateReceiveCount tracking +# across message redeliveries. + +require 'concurrent' + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' }) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +# Atomic counter for fail tracking +fail_counter = Concurrent::AtomicFixnum.new(2) + +worker_class = Class.new do + include Shoryuken::Worker + + shoryuken_options auto_delete: false, batch: false + + define_method(:perform) do |sqs_msg, body| + receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i + DT[:receive_counts] << receive_count + + if fail_counter.value > 0 + fail_counter.decrement + raise "Simulated failure" + else + sqs_msg.delete + end + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +Shoryuken.register_worker(queue_name, worker_class) + +Shoryuken::Client.queues(queue_name).send_message(message_body: 'retry-count-test') + +poll_queues_until(timeout: 20) { DT[:receive_counts].size >= 3 } + +assert(DT[:receive_counts].size >= 3) +assert_equal(DT[:receive_counts], DT[:receive_counts].sort, "Receive counts should be increasing") +assert_equal(1, DT[:receive_counts].first) diff --git a/spec/integration/spec_helper.rb b/spec/integration/spec_helper.rb new file mode 100644 index 00000000..36a0594a --- /dev/null +++ b/spec/integration/spec_helper.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# Integration test spec helper +# This file is auto-required by RSpec for integration tests + +require 'shoryuken' +require_relative '../integrations_helper' diff --git a/spec/integration/visibility_timeout/visibility_timeout_spec.rb b/spec/integration/visibility_timeout/visibility_timeout_spec.rb new file mode 100644 index 00000000..14bbb139 --- /dev/null +++ b/spec/integration/visibility_timeout/visibility_timeout_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# This spec tests visibility timeout management including manual visibility +# extension during long processing. + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '5' }) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + def perform(sqs_msg, body) + # Extend visibility before long processing + sqs_msg.change_visibility(visibility_timeout: 30) + DT[:visibility_extended] << true + + sleep 2 # Simulate slow processing + + DT[:messages] << body + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +worker_class.get_shoryuken_options['auto_delete'] = true +worker_class.get_shoryuken_options['batch'] = false +Shoryuken.register_worker(queue_name, worker_class) + +Shoryuken::Client.queues(queue_name).send_message(message_body: 'extend-test') + +poll_queues_until { DT[:messages].size >= 1 } + +assert_equal(1, DT[:messages].size) +assert(DT[:visibility_extended].any?, "Expected visibility to be extended") diff --git a/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb b/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb new file mode 100644 index 00000000..ae2664cf --- /dev/null +++ b/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# This spec tests worker lifecycle including worker registration and discovery. + +setup_localstack + +queue_name = DT.queue +create_test_queue(queue_name) +Shoryuken.add_group('default', 1) +Shoryuken.add_queue(queue_name, 1, 'default') + +worker_class = Class.new do + include Shoryuken::Worker + + def perform(sqs_msg, body) + DT[:messages] << body + end +end + +worker_class.get_shoryuken_options['queue'] = queue_name +worker_class.get_shoryuken_options['auto_delete'] = true +worker_class.get_shoryuken_options['batch'] = false +Shoryuken.register_worker(queue_name, worker_class) + +registered = Shoryuken.worker_registry.workers(queue_name) +assert_includes(registered, worker_class) + +Shoryuken::Client.queues(queue_name).send_message(message_body: 'lifecycle-test') + +poll_queues_until { DT[:messages].size >= 1 } + +assert_equal(1, DT[:messages].size) +assert_equal('lifecycle-test', DT[:messages].first) diff --git a/spec/integrations_helper.rb b/spec/integrations_helper.rb new file mode 100644 index 00000000..beb9c2c9 --- /dev/null +++ b/spec/integrations_helper.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +# Integration test helper for process-isolated testing + +require 'timeout' +require 'json' +require 'securerandom' +require 'aws-sdk-sqs' +require 'shoryuken' +require 'singleton' + +# Thread-safe data collector for integration tests +# Inspired by Karafka's DataCollector pattern +# Usage: DT[:key] << value, DT[:key].size, DT.clear +class DataCollector + include Singleton + + MUTEX = Mutex.new + private_constant :MUTEX + + attr_reader :queues, :data + + class << self + def queue + instance.queue + end + + def queues + instance.queues + end + + def data + instance.data + end + + def [](key) + MUTEX.synchronize { data[key] } + end + + def []=(key, value) + MUTEX.synchronize { data[key] = value } + end + + def uuids(amount) + Array.new(amount) { uuid } + end + + def uuid + "it-#{SecureRandom.uuid[0, 8]}" + end + + def clear + MUTEX.synchronize { instance.clear } + end + + def key?(key) + instance.data.key?(key) + end + end + + def initialize + @mutex = Mutex.new + @queues = Array.new(100) { "it-#{SecureRandom.hex(6)}" } + @data = Hash.new do |hash, key| + @mutex.synchronize do + break hash[key] if hash.key?(key) + + hash[key] = [] + end + end + end + + def queue + queues.first + end + + def clear + @mutex.synchronize do + @queues.clear + @queues.concat(Array.new(100) { "it-#{SecureRandom.hex(6)}" }) + @data.clear + end + end +end + +# Short alias for DataCollector +DT = DataCollector + +module IntegrationsHelper + class TestFailure < StandardError; end + + # Assertions + def assert(condition, message = "Assertion failed") + raise TestFailure, message unless condition + end + + def assert_equal(expected, actual, message = nil) + message ||= "Expected #{expected.inspect}, got #{actual.inspect}" + assert(expected == actual, message) + end + + def assert_includes(collection, item, message = nil) + message ||= "Expected #{collection.inspect} to include #{item.inspect}" + assert(collection.include?(item), message) + end + + def refute(condition, message = "Refutation failed") + assert(!condition, message) + end + + # Configure ActiveJob with Shoryuken adapter + def setup_active_job + require 'active_job' + require 'active_job/queue_adapters/shoryuken_adapter' + require 'active_job/extensions' + + ActiveJob::Base.queue_adapter = :shoryuken + end + + # Configure Shoryuken to use LocalStack for real SQS integration tests + def setup_localstack + Aws.config[:stub_responses] = false + + sqs_client = Aws::SQS::Client.new( + region: 'us-east-1', + endpoint: 'http://localhost:4566', + access_key_id: 'fake', + secret_access_key: 'fake' + ) + + Shoryuken.options[:concurrency] = 25 + Shoryuken.options[:delay] = 0 + Shoryuken.options[:timeout] = 8 + + executor = Concurrent::CachedThreadPool.new(auto_terminate: true) + Shoryuken.define_singleton_method(:launcher_executor) { executor } + + Shoryuken.configure_client { |config| config.sqs_client = sqs_client } + Shoryuken.configure_server { |config| config.sqs_client = sqs_client } + end + + # Queue helpers + def create_test_queue(queue_name, attributes: {}) + Shoryuken::Client.sqs.create_queue(queue_name: queue_name, attributes: attributes) + end + + def delete_test_queue(queue_name) + queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url + Shoryuken::Client.sqs.delete_queue(queue_url: queue_url) + rescue Aws::SQS::Errors::NonExistentQueue + end + + def create_fifo_queue(queue_name) + create_test_queue(queue_name, attributes: { + 'FifoQueue' => 'true', + 'ContentBasedDeduplication' => 'true' + }) + end + + # Poll until condition met + def poll_queues_until(timeout: 15) + launcher = Shoryuken::Launcher.new + launcher.start + Timeout.timeout(timeout) { sleep 0.5 until yield } + ensure + launcher.stop + end + + # Simple mock object + def double(_name = nil) + Object.new + end + + # Job capture for ActiveJob tests + class JobCapture + attr_reader :jobs + + def initialize + @jobs = [] + end + + def start_capturing + @jobs.clear + capture = self + + queue_mock = Object.new + queue_mock.define_singleton_method(:fifo?) { false } + queue_mock.define_singleton_method(:send_message) do |params| + capture.jobs << { + queue: params[:queue_name] || :default, + message_body: params[:message_body], + delay_seconds: params[:delay_seconds], + message_attributes: params[:message_attributes] + } + end + + Shoryuken::Client.define_singleton_method(:queues) do |queue_name = nil| + queue_mock.define_singleton_method(:name) { queue_name } if queue_name + queue_name ? queue_mock : { default: queue_mock } + end + + Shoryuken.define_singleton_method(:register_worker) { |*| nil } + end + + def last_job + @jobs.last + end + + def job_count + @jobs.size + end + end +end + +include IntegrationsHelper diff --git a/spec/lib/active_job/extensions_spec.rb b/spec/lib/active_job/extensions_spec.rb new file mode 100644 index 00000000..d9ad0eba --- /dev/null +++ b/spec/lib/active_job/extensions_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# Skip this spec if ActiveSupport is not available, as the extensions require it +if defined?(ActiveSupport) + require 'active_job/extensions' + + RSpec.describe Shoryuken::ActiveJob do + describe Shoryuken::ActiveJob::SQSSendMessageParametersAccessor do + let(:job_class) do + Class.new do + include Shoryuken::ActiveJob::SQSSendMessageParametersAccessor + end + end + + let(:job_instance) { job_class.new } + + describe 'included behavior' do + it 'adds sqs_send_message_parameters accessor' do + expect(job_instance).to respond_to(:sqs_send_message_parameters) + expect(job_instance).to respond_to(:sqs_send_message_parameters=) + end + + it 'allows setting and getting sqs_send_message_parameters' do + params = { message_group_id: 'group1', message_deduplication_id: 'dedup1' } + job_instance.sqs_send_message_parameters = params + expect(job_instance.sqs_send_message_parameters).to eq(params) + end + end + end + + describe Shoryuken::ActiveJob::SQSSendMessageParametersSupport do + let(:base_class) do + Class.new do + attr_accessor :sqs_send_message_parameters + + def initialize(*arguments) + # Mock ActiveJob::Base initialization + end + + def enqueue(options = {}) + # Mock ActiveJob::Base enqueue method that returns remaining options + options + end + end + end + + let(:job_class) do + Class.new(base_class) do + prepend Shoryuken::ActiveJob::SQSSendMessageParametersSupport + end + end + + describe '#initialize' do + it 'initializes sqs_send_message_parameters to empty hash' do + job = job_class.new('arg1', 'arg2') + expect(job.sqs_send_message_parameters).to eq({}) + end + + it 'calls super with the provided arguments' do + expect_any_instance_of(base_class).to receive(:initialize).with('arg1', 'arg2') + job_class.new('arg1', 'arg2') + end + + it 'handles ruby2_keywords compatibility' do + # Test that ruby2_keywords is called if available + if respond_to?(:ruby2_keywords, true) + expect(job_class.method(:new)).to respond_to(:ruby2_keywords) if RUBY_VERSION >= '2.7' + end + end + end + + describe '#enqueue' do + let(:job_instance) { job_class.new } + + it 'extracts SQS-specific options and merges them into sqs_send_message_parameters' do + options = { + wait: 5 * 60, # 5 minutes in seconds + message_attributes: { 'type' => 'important' }, + message_system_attributes: { 'source' => 'api' }, + message_deduplication_id: 'dedup123', + message_group_id: 'group456', + other_option: 'value' + } + + remaining_options = job_instance.enqueue(options) + + expect(job_instance.sqs_send_message_parameters).to eq({ + message_attributes: { 'type' => 'important' }, + message_system_attributes: { 'source' => 'api' }, + message_deduplication_id: 'dedup123', + message_group_id: 'group456' + }) + + expect(remaining_options).to eq({ + wait: 300, + other_option: 'value' + }) + end + + it 'handles empty options gracefully' do + remaining_options = job_instance.enqueue({}) + expect(job_instance.sqs_send_message_parameters).to eq({}) + expect(remaining_options).to eq({}) + end + + it 'merges new SQS options with existing ones' do + job_instance.sqs_send_message_parameters = { message_group_id: 'existing_group' } + + options = { message_deduplication_id: 'new_dedup' } + job_instance.enqueue(options) + + expect(job_instance.sqs_send_message_parameters).to eq({ + message_group_id: 'existing_group', + message_deduplication_id: 'new_dedup' + }) + end + + it 'overwrites existing SQS options when the same key is provided' do + job_instance.sqs_send_message_parameters = { message_group_id: 'old_group' } + + options = { message_group_id: 'new_group' } + job_instance.enqueue(options) + + expect(job_instance.sqs_send_message_parameters).to eq({ + message_group_id: 'new_group' + }) + end + end + end + + describe 'module constants' do + it 'defines SQSSendMessageParametersAccessor' do + expect(Shoryuken::ActiveJob::SQSSendMessageParametersAccessor).to be_a(Module) + end + + it 'defines SQSSendMessageParametersSupport' do + expect(Shoryuken::ActiveJob::SQSSendMessageParametersSupport).to be_a(Module) + end + end + end +else + RSpec.describe 'Shoryuken::ActiveJob (skipped - ActiveSupport not available)' do + it 'skips tests when ActiveSupport is not available' do + skip('ActiveSupport not available in test environment') + end + end +end \ No newline at end of file diff --git a/spec/lib/active_job/queue_adapters/shoryuken_adapter_spec.rb b/spec/lib/active_job/queue_adapters/shoryuken_adapter_spec.rb new file mode 100644 index 00000000..073fe2e2 --- /dev/null +++ b/spec/lib/active_job/queue_adapters/shoryuken_adapter_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'shared_examples_for_active_job' +require 'active_job/queue_adapters/shoryuken_adapter' +require 'active_support/core_ext/numeric/time' + +RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do + include_examples 'active_job_adapters' + + describe '#enqueue_after_transaction_commit?' do + it 'returns true to support Rails 7.2+ transaction commit behavior' do + adapter = described_class.new + expect(adapter.enqueue_after_transaction_commit?).to eq(true) + end + end + + describe '.instance' do + it 'returns the same instance (singleton pattern)' do + instance1 = described_class.instance + instance2 = described_class.instance + expect(instance1).to be(instance2) + end + + it 'returns a ShoryukenAdapter instance' do + expect(described_class.instance).to be_a(described_class) + end + end + +end \ No newline at end of file diff --git a/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb b/spec/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter_spec.rb similarity index 89% rename from spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb rename to spec/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter_spec.rb index a6a68733..f9984aeb 100644 --- a/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +++ b/spec/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true require 'shared_examples_for_active_job' -require 'shoryuken/extensions/active_job_adapter' -require 'shoryuken/extensions/active_job_concurrent_send_adapter' +require 'active_job/queue_adapters/shoryuken_adapter' +require 'active_job/queue_adapters/shoryuken_concurrent_send_adapter' RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do include_examples 'active_job_adapters' @@ -37,4 +37,4 @@ subject.enqueue(job, options) end end -end +end \ No newline at end of file diff --git a/spec/shoryuken/extensions/active_job_wrapper_spec.rb b/spec/lib/shoryuken/active_job/job_wrapper_spec.rb similarity index 75% rename from spec/shoryuken/extensions/active_job_wrapper_spec.rb rename to spec/lib/shoryuken/active_job/job_wrapper_spec.rb index 40fb6878..dcd2b282 100644 --- a/spec/shoryuken/extensions/active_job_wrapper_spec.rb +++ b/spec/lib/shoryuken/active_job/job_wrapper_spec.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true require 'active_job' -require 'shoryuken/extensions/active_job_extensions' -require 'shoryuken/extensions/active_job_adapter' +require 'active_job/extensions' +require 'active_job/queue_adapters/shoryuken_adapter' -RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper do +RSpec.describe Shoryuken::ActiveJob::JobWrapper do subject { described_class.new } describe '#perform' do @@ -18,4 +18,4 @@ subject.perform sqs_msg, job_hash end end -end +end \ No newline at end of file diff --git a/spec/shoryuken/body_parser_spec.rb b/spec/lib/shoryuken/body_parser_spec.rb similarity index 100% rename from spec/shoryuken/body_parser_spec.rb rename to spec/lib/shoryuken/body_parser_spec.rb diff --git a/spec/shoryuken/client_spec.rb b/spec/lib/shoryuken/client_spec.rb similarity index 100% rename from spec/shoryuken/client_spec.rb rename to spec/lib/shoryuken/client_spec.rb diff --git a/spec/shoryuken/default_exception_handler_spec.rb b/spec/lib/shoryuken/default_exception_handler_spec.rb similarity index 100% rename from spec/shoryuken/default_exception_handler_spec.rb rename to spec/lib/shoryuken/default_exception_handler_spec.rb diff --git a/spec/shoryuken/default_worker_registry_spec.rb b/spec/lib/shoryuken/default_worker_registry_spec.rb similarity index 100% rename from spec/shoryuken/default_worker_registry_spec.rb rename to spec/lib/shoryuken/default_worker_registry_spec.rb diff --git a/spec/shoryuken/environment_loader_spec.rb b/spec/lib/shoryuken/environment_loader_spec.rb similarity index 100% rename from spec/shoryuken/environment_loader_spec.rb rename to spec/lib/shoryuken/environment_loader_spec.rb diff --git a/spec/shoryuken/fetcher_spec.rb b/spec/lib/shoryuken/fetcher_spec.rb similarity index 100% rename from spec/shoryuken/fetcher_spec.rb rename to spec/lib/shoryuken/fetcher_spec.rb diff --git a/spec/shoryuken/helpers/atomic_boolean_spec.rb b/spec/lib/shoryuken/helpers/atomic_boolean_spec.rb similarity index 100% rename from spec/shoryuken/helpers/atomic_boolean_spec.rb rename to spec/lib/shoryuken/helpers/atomic_boolean_spec.rb diff --git a/spec/shoryuken/helpers/atomic_counter_spec.rb b/spec/lib/shoryuken/helpers/atomic_counter_spec.rb similarity index 100% rename from spec/shoryuken/helpers/atomic_counter_spec.rb rename to spec/lib/shoryuken/helpers/atomic_counter_spec.rb diff --git a/spec/shoryuken/helpers/atomic_hash_spec.rb b/spec/lib/shoryuken/helpers/atomic_hash_spec.rb similarity index 100% rename from spec/shoryuken/helpers/atomic_hash_spec.rb rename to spec/lib/shoryuken/helpers/atomic_hash_spec.rb diff --git a/spec/shoryuken/helpers/hash_utils_spec.rb b/spec/lib/shoryuken/helpers/hash_utils_spec.rb similarity index 97% rename from spec/shoryuken/helpers/hash_utils_spec.rb rename to spec/lib/shoryuken/helpers/hash_utils_spec.rb index 7153ba7c..139bd8f2 100644 --- a/spec/shoryuken/helpers/hash_utils_spec.rb +++ b/spec/lib/shoryuken/helpers/hash_utils_spec.rb @@ -7,21 +7,21 @@ it 'converts string keys to symbols' do input = { 'key1' => 'value1', 'key2' => 'value2' } expected = { key1: 'value1', key2: 'value2' } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end it 'leaves symbol keys unchanged' do input = { key1: 'value1', key2: 'value2' } expected = { key1: 'value1', key2: 'value2' } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end it 'handles mixed key types' do input = { 'string_key' => 'value1', :symbol_key => 'value2' } expected = { string_key: 'value1', symbol_key: 'value2' } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end @@ -35,7 +35,7 @@ }, 'top_level' => 'value' } - + expected = { level1: { level2: { @@ -45,7 +45,7 @@ }, top_level: 'value' } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end @@ -58,7 +58,7 @@ 'metadata' => nil } } - + expected = { config: { timeout: 30, @@ -67,7 +67,7 @@ metadata: nil } } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end @@ -78,7 +78,7 @@ it 'handles hash with empty nested hash' do input = { 'key' => {} } expected = { key: {} } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end @@ -94,10 +94,10 @@ # Create a key that will raise an exception when converted to symbol problematic_key = Object.new allow(problematic_key).to receive(:to_sym).and_raise(StandardError) - + input = { problematic_key => 'value', 'normal_key' => 'normal_value' } result = described_class.deep_symbolize_keys(input) - + # The problematic key should remain as-is, normal key should be symbolized expect(result[problematic_key]).to eq('value') expect(result[:normal_key]).to eq('normal_value') @@ -106,9 +106,9 @@ it 'does not modify the original hash' do input = { 'key' => { 'nested' => 'value' } } original_input = input.dup - + described_class.deep_symbolize_keys(input) - + expect(input).to eq(original_input) end @@ -125,7 +125,7 @@ 'mailers' => { 'concurrency' => 2 } } } - + expected = { database: { host: 'localhost', @@ -137,7 +137,7 @@ mailers: { concurrency: 2 } } } - + expect(described_class.deep_symbolize_keys(input)).to eq(expected) end end diff --git a/spec/shoryuken/helpers/string_utils_spec.rb b/spec/lib/shoryuken/helpers/string_utils_spec.rb similarity index 99% rename from spec/shoryuken/helpers/string_utils_spec.rb rename to spec/lib/shoryuken/helpers/string_utils_spec.rb index c25fa598..04568462 100644 --- a/spec/shoryuken/helpers/string_utils_spec.rb +++ b/spec/lib/shoryuken/helpers/string_utils_spec.rb @@ -74,7 +74,7 @@ unless Object.const_defined?('MyApp') Object.const_set('MyApp', Module.new) end - + unless MyApp.const_defined?('EmailWorker') MyApp.const_set('EmailWorker', Class.new) end @@ -107,9 +107,9 @@ it 'handles single character constant names' do # Define a single character constant for testing Object.const_set('A', Class.new) unless Object.const_defined?('A') - + expect(described_class.constantize('A')).to eq(A) - + # Clean up Object.send(:remove_const, 'A') if Object.const_defined?('A') end diff --git a/spec/shoryuken/helpers/timer_task_spec.rb b/spec/lib/shoryuken/helpers/timer_task_spec.rb similarity index 100% rename from spec/shoryuken/helpers/timer_task_spec.rb rename to spec/lib/shoryuken/helpers/timer_task_spec.rb diff --git a/spec/shoryuken/helpers_integration_spec.rb b/spec/lib/shoryuken/helpers_integration_spec.rb similarity index 98% rename from spec/shoryuken/helpers_integration_spec.rb rename to spec/lib/shoryuken/helpers_integration_spec.rb index ef8b5272..602befb0 100644 --- a/spec/shoryuken/helpers_integration_spec.rb +++ b/spec/lib/shoryuken/helpers_integration_spec.rb @@ -2,7 +2,7 @@ RSpec.describe 'Helpers Integration' do # Integration tests for helper utility methods that replaced core extensions - + describe Shoryuken::Helpers::HashUtils do describe '.deep_symbolize_keys' do it 'converts keys into symbols recursively' do @@ -12,12 +12,12 @@ 'key31' => { 'key311' => 'value311' }, 'key32' => 'value32' } } - + expected = { key1: 'value1', key2: 'value2', key3: { key31: { key311: 'value311' }, key32: 'value32' } } - + expect(Shoryuken::Helpers::HashUtils.deep_symbolize_keys(input)).to eq(expected) end @@ -34,7 +34,7 @@ it 'handles mixed value types' do input = { 'key1' => 'string', 'key2' => 123, 'key3' => { 'nested' => true } } expected = { key1: 'string', key2: 123, key3: { nested: true } } - + expect(Shoryuken::Helpers::HashUtils.deep_symbolize_keys(input)).to eq(expected) end end @@ -43,7 +43,7 @@ describe Shoryuken::Helpers::StringUtils do describe '.constantize' do class HelloWorld; end - + it 'returns a class from a string' do expect(Shoryuken::Helpers::StringUtils.constantize('HelloWorld')).to eq(HelloWorld) end @@ -75,20 +75,20 @@ class HelloWorld; end 'mailers' => { 'worker_class' => 'String' } } } - + symbolized = Shoryuken::Helpers::HashUtils.deep_symbolize_keys(config_data) - + expect(symbolized).to eq({ queues: { default: { worker_class: 'Object' }, mailers: { worker_class: 'String' } } }) - + # Test constantizing the worker classes default_worker = Shoryuken::Helpers::StringUtils.constantize(symbolized[:queues][:default][:worker_class]) mailer_worker = Shoryuken::Helpers::StringUtils.constantize(symbolized[:queues][:mailers][:worker_class]) - + expect(default_worker).to eq(Object) expect(mailer_worker).to eq(String) end diff --git a/spec/shoryuken/inline_message_spec.rb b/spec/lib/shoryuken/inline_message_spec.rb similarity index 100% rename from spec/shoryuken/inline_message_spec.rb rename to spec/lib/shoryuken/inline_message_spec.rb diff --git a/spec/shoryuken/launcher_spec.rb b/spec/lib/shoryuken/launcher_spec.rb similarity index 100% rename from spec/shoryuken/launcher_spec.rb rename to spec/lib/shoryuken/launcher_spec.rb diff --git a/spec/lib/shoryuken/logging_spec.rb b/spec/lib/shoryuken/logging_spec.rb new file mode 100644 index 00000000..c6df94c6 --- /dev/null +++ b/spec/lib/shoryuken/logging_spec.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Shoryuken::Logging do + describe Shoryuken::Logging::Base do + let(:formatter) { described_class.new } + + describe '#tid' do + it 'returns a string representing the thread ID' do + expect(formatter.tid).to be_a(String) + end + + it 'returns the same value for the same thread' do + tid1 = formatter.tid + tid2 = formatter.tid + expect(tid1).to eq(tid2) + end + + it 'caches the thread ID in thread-local storage' do + tid = formatter.tid + expect(Thread.current['shoryuken_tid']).to eq(tid) + end + end + + describe '#context' do + it 'returns empty string when no context is set' do + Thread.current[:shoryuken_context] = nil + expect(formatter.context).to eq('') + end + + it 'returns formatted context when context is set' do + Thread.current[:shoryuken_context] = 'test_context' + expect(formatter.context).to eq(' test_context') + end + end + end + + describe Shoryuken::Logging::Pretty do + let(:formatter) { described_class.new } + let(:time) { Time.new(2023, 8, 15, 10, 30, 45, '+00:00') } + + describe '#call' do + it 'formats log messages with timestamp' do + allow(formatter).to receive(:tid).and_return('abc123') + Thread.current[:shoryuken_context] = nil + + result = formatter.call('INFO', time, 'program', 'test message') + expect(result).to eq("2023-08-15T10:30:45Z #{Process.pid} TID-abc123 INFO: test message\n") + end + + it 'includes context when present' do + allow(formatter).to receive(:tid).and_return('abc123') + Thread.current[:shoryuken_context] = 'worker-1' + + result = formatter.call('ERROR', time, 'program', 'error message') + expect(result).to eq("2023-08-15T10:30:45Z #{Process.pid} TID-abc123 worker-1 ERROR: error message\n") + end + end + end + + describe Shoryuken::Logging::WithoutTimestamp do + let(:formatter) { described_class.new } + + describe '#call' do + it 'formats log messages without timestamp' do + allow(formatter).to receive(:tid).and_return('xyz789') + Thread.current[:shoryuken_context] = nil + + result = formatter.call('DEBUG', Time.now, 'program', 'debug message') + expect(result).to eq("pid=#{Process.pid} tid=xyz789 DEBUG: debug message\n") + end + + it 'includes context when present' do + allow(formatter).to receive(:tid).and_return('xyz789') + Thread.current[:shoryuken_context] = 'queue-processor' + + result = formatter.call('WARN', Time.now, 'program', 'warning message') + expect(result).to eq("pid=#{Process.pid} tid=xyz789 queue-processor WARN: warning message\n") + end + end + end + + describe '.with_context' do + it 'sets context for the duration of the block' do + described_class.with_context('test_context') do + expect(Thread.current[:shoryuken_context]).to eq('test_context') + end + end + + it 'clears context after the block completes' do + described_class.with_context('test_context') do + # context is set + end + expect(Thread.current[:shoryuken_context]).to be_nil + end + + it 'clears context even when an exception is raised' do + expect do + described_class.with_context('test_context') do + raise StandardError, 'test error' + end + end.to raise_error(StandardError, 'test error') + + expect(Thread.current[:shoryuken_context]).to be_nil + end + + it 'returns the value of the block' do + result = described_class.with_context('test_context') do + 'block_result' + end + expect(result).to eq('block_result') + end + end + + describe '.initialize_logger' do + it 'creates a new Logger instance' do + logger = described_class.initialize_logger + expect(logger).to be_a(Logger) + end + + it 'sets default log level to INFO' do + logger = described_class.initialize_logger + expect(logger.level).to eq(Logger::INFO) + end + + it 'uses Pretty formatter by default' do + logger = described_class.initialize_logger + expect(logger.formatter).to be_a(Shoryuken::Logging::Pretty) + end + + it 'accepts custom log target' do + log_target = StringIO.new + logger = described_class.initialize_logger(log_target) + expect(logger.instance_variable_get(:@logdev).dev).to eq(log_target) + end + end + + describe '.logger' do + after do + # Reset the instance variable to avoid affecting other tests + described_class.instance_variable_set(:@logger, nil) + end + + it 'returns a logger instance' do + expect(described_class.logger).to be_a(Logger) + end + + it 'memoizes the logger instance' do + logger1 = described_class.logger + logger2 = described_class.logger + expect(logger1).to be(logger2) + end + + it 'initializes logger if not already set' do + expect(described_class).to receive(:initialize_logger).and_call_original + described_class.logger + end + end + + describe '.logger=' do + after do + # Reset the instance variable to avoid affecting other tests + described_class.instance_variable_set(:@logger, nil) + end + + it 'sets the logger instance' do + custom_logger = Logger.new('/dev/null') + described_class.logger = custom_logger + expect(described_class.logger).to be(custom_logger) + end + + it 'sets null logger when passed nil' do + described_class.logger = nil + logger = described_class.logger + # The logger should be configured to output to /dev/null + expect(logger).to be_a(Logger) + end + end +end \ No newline at end of file diff --git a/spec/shoryuken/manager_spec.rb b/spec/lib/shoryuken/manager_spec.rb similarity index 100% rename from spec/shoryuken/manager_spec.rb rename to spec/lib/shoryuken/manager_spec.rb diff --git a/spec/lib/shoryuken/message_spec.rb b/spec/lib/shoryuken/message_spec.rb new file mode 100644 index 00000000..06c618c5 --- /dev/null +++ b/spec/lib/shoryuken/message_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Shoryuken::Message do + let(:client) { instance_double('Aws::SQS::Client') } + let(:queue) { instance_double('Shoryuken::Queue', name: 'test-queue', url: 'https://sqs.us-east-1.amazonaws.com/123456789/test-queue') } + let(:data) do + instance_double('Aws::SQS::Types::Message', + message_id: 'msg-123', + receipt_handle: 'handle-456', + md5_of_body: 'abcd1234', + body: '{"test": "data"}', + attributes: { 'ApproximateReceiveCount' => '1' }, + md5_of_message_attributes: 'efgh5678', + message_attributes: { 'type' => 'test' }) + end + + subject { described_class.new(client, queue, data) } + + describe '#initialize' do + it 'sets client, queue_url, queue_name, and data' do + expect(subject.client).to eq(client) + expect(subject.queue_url).to eq('https://sqs.us-east-1.amazonaws.com/123456789/test-queue') + expect(subject.queue_name).to eq('test-queue') + expect(subject.data).to eq(data) + end + end + + describe 'delegated methods' do + it 'delegates message_id to data' do + expect(subject.message_id).to eq('msg-123') + end + + it 'delegates receipt_handle to data' do + expect(subject.receipt_handle).to eq('handle-456') + end + + it 'delegates md5_of_body to data' do + expect(subject.md5_of_body).to eq('abcd1234') + end + + it 'delegates body to data' do + expect(subject.body).to eq('{"test": "data"}') + end + + it 'delegates attributes to data' do + expect(subject.attributes).to eq({ 'ApproximateReceiveCount' => '1' }) + end + + it 'delegates md5_of_message_attributes to data' do + expect(subject.md5_of_message_attributes).to eq('efgh5678') + end + + it 'delegates message_attributes to data' do + expect(subject.message_attributes).to eq({ 'type' => 'test' }) + end + end + + describe '#delete' do + it 'calls delete_message on the client with correct parameters' do + expect(client).to receive(:delete_message).with( + queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789/test-queue', + receipt_handle: 'handle-456' + ) + + subject.delete + end + end + + describe '#change_visibility' do + it 'calls change_message_visibility on the client with merged parameters' do + options = { visibility_timeout: 300 } + + expect(client).to receive(:change_message_visibility).with(hash_including( + visibility_timeout: 300, + queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789/test-queue', + receipt_handle: 'handle-456' + )) + + subject.change_visibility(options) + end + + it 'merges queue_url and receipt_handle into provided options' do + options = { visibility_timeout: 120, custom_param: 'value' } + + expect(client).to receive(:change_message_visibility).with(hash_including( + visibility_timeout: 120, + custom_param: 'value', + queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789/test-queue', + receipt_handle: 'handle-456' + )) + + subject.change_visibility(options) + end + end + + describe '#visibility_timeout=' do + it 'calls change_message_visibility on the client with the timeout' do + expect(client).to receive(:change_message_visibility).with( + queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789/test-queue', + receipt_handle: 'handle-456', + visibility_timeout: 600 + ) + + subject.visibility_timeout = 600 + end + end +end \ No newline at end of file diff --git a/spec/shoryuken/middleware/chain_spec.rb b/spec/lib/shoryuken/middleware/chain_spec.rb similarity index 100% rename from spec/shoryuken/middleware/chain_spec.rb rename to spec/lib/shoryuken/middleware/chain_spec.rb diff --git a/spec/lib/shoryuken/middleware/entry_spec.rb b/spec/lib/shoryuken/middleware/entry_spec.rb new file mode 100644 index 00000000..a08e36d1 --- /dev/null +++ b/spec/lib/shoryuken/middleware/entry_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'shoryuken/middleware/entry' + +RSpec.describe Shoryuken::Middleware::Entry do + describe '#initialize' do + it 'stores the middleware class' do + entry = described_class.new(String) + expect(entry.klass).to eq String + end + + it 'stores initialization arguments' do + entry = described_class.new(String, 'arg1', 'arg2') + expect(entry.instance_variable_get(:@args)).to eq ['arg1', 'arg2'] + end + end + + describe '#make_new' do + let(:test_class) do + Class.new do + attr_reader :args + + def initialize(*args) + @args = args + end + end + end + + it 'creates a new instance of the stored class without arguments' do + entry = described_class.new(test_class) + instance = entry.make_new + + expect(instance).to be_a test_class + expect(instance.args).to eq [] + end + + it 'creates a new instance with stored arguments' do + entry = described_class.new(test_class, 'arg1', 42, { key: 'value' }) + instance = entry.make_new + + expect(instance).to be_a test_class + expect(instance.args).to eq ['arg1', 42, { key: 'value' }] + end + + it 'creates a new instance each time it is called' do + entry = described_class.new(test_class, 'shared_arg') + instance1 = entry.make_new + instance2 = entry.make_new + + expect(instance1).to be_a test_class + expect(instance2).to be_a test_class + expect(instance1).not_to be instance2 + expect(instance1.args).to eq instance2.args + end + end + + describe '#klass' do + it 'returns the stored class' do + entry = described_class.new(Array) + expect(entry.klass).to eq Array + end + + it 'is readable' do + entry = described_class.new(Hash) + expect(entry.klass).to eq Hash + end + end +end \ No newline at end of file diff --git a/spec/lib/shoryuken/middleware/server/active_record_spec.rb b/spec/lib/shoryuken/middleware/server/active_record_spec.rb new file mode 100644 index 00000000..b01b7064 --- /dev/null +++ b/spec/lib/shoryuken/middleware/server/active_record_spec.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Shoryuken::Middleware::Server::ActiveRecord do + subject { described_class.new } + + # Mock ActiveRecord to avoid requiring the actual gem in tests + before do + # Create mock ActiveRecord module + active_record_module = Module.new + + # Create mock Base class with simplified methods + active_record_base = Class.new do + @connection_handler = nil + + def self.clear_active_connections! + # Mock implementation for Rails < 7.1 + end + + def self.connection_handler + @connection_handler ||= Object.new.tap do |handler| + def handler.clear_active_connections!(_pool_key) + # Mock implementation for Rails 7.1+ + end + end + end + end + + active_record_module.const_set('Base', active_record_base) + stub_const('ActiveRecord', active_record_module) + + # Mock version checking - start with a simple approach + def active_record_module.version + @version ||= Object.new.tap do |v| + def v.>=(other) + # For our tests, we'll control this with instance variables + @is_rails_71_or_higher ||= false + end + + def v.rails_71_or_higher! + @is_rails_71_or_higher = true + end + + def v.rails_70! + @is_rails_71_or_higher = false + end + end + end + + # Mock Gem::Version + unless defined?(Gem::Version) + gem_module = Module.new + gem_version_class = Class.new do + def initialize(_version) + # Simple mock + end + end + gem_module.const_set('Version', gem_version_class) + stub_const('Gem', gem_module) + end + end + + describe '#call' do + it 'yields to the block' do + block_called = false + subject.call do + block_called = true + end + expect(block_called).to be true + end + + it 'returns the value from the block' do + result = subject.call { 'block_result' } + expect(result).to eq('block_result') + end + + context 'when ActiveRecord version is 7.1 or higher' do + before do + # Mock Rails 7.1+ behavior + allow(ActiveRecord).to receive(:version).and_return(double('>=' => true)) + end + + it 'calls clear_active_connections! on connection_handler with :all parameter' do + connection_handler = ActiveRecord::Base.connection_handler + expect(connection_handler).to receive(:clear_active_connections!).with(:all) + + subject.call { 'test' } + end + + it 'clears connections even when an exception is raised' do + connection_handler = ActiveRecord::Base.connection_handler + expect(connection_handler).to receive(:clear_active_connections!).with(:all) + + expect do + subject.call { raise StandardError, 'test error' } + end.to raise_error(StandardError, 'test error') + end + end + + context 'when ActiveRecord version is lower than 7.1' do + before do + # Mock Rails < 7.1 behavior + allow(ActiveRecord).to receive(:version).and_return(double('>=' => false)) + end + + it 'calls clear_active_connections! directly on ActiveRecord::Base' do + expect(ActiveRecord::Base).to receive(:clear_active_connections!) + + subject.call { 'test' } + end + + it 'clears connections even when an exception is raised' do + expect(ActiveRecord::Base).to receive(:clear_active_connections!) + + expect do + subject.call { raise StandardError, 'test error' } + end.to raise_error(StandardError, 'test error') + end + end + + it 'works with middleware arguments (ignores them)' do + allow(ActiveRecord).to receive(:version).and_return(double('>=' => false)) + expect(ActiveRecord::Base).to receive(:clear_active_connections!) + + worker = double('worker') + message = double('message') + + result = subject.call(worker, message) { 'middleware_result' } + expect(result).to eq('middleware_result') + end + end +end \ No newline at end of file diff --git a/spec/shoryuken/middleware/server/auto_delete_spec.rb b/spec/lib/shoryuken/middleware/server/auto_delete_spec.rb similarity index 100% rename from spec/shoryuken/middleware/server/auto_delete_spec.rb rename to spec/lib/shoryuken/middleware/server/auto_delete_spec.rb diff --git a/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb b/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb similarity index 54% rename from spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb rename to spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb index 8b028324..e2755f0b 100644 --- a/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +++ b/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb @@ -64,4 +64,54 @@ def run_and_raise(worker, queue, sqs_msg) Runner.new.run_and_sleep(TestWorker.new, queue, sqs_msg, visibility_timeout) end + + context 'when batch worker with auto_visibility_timeout' do + it 'warns and does not extend visibility for batch workers' do + TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true + + expect(Shoryuken.logger).to receive(:warn) do |&block| + expect(block.call).to include("Auto extend visibility isn't supported for batch workers") + end + + expect { |b| subject.call(TestWorker.new, queue, [sqs_msg], nil, &b) }.to yield_control + end + end + + context 'when visibility extension fails' do + it 'logs error when change_visibility raises an exception' do + TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true + + allow(sqs_msg).to receive(:queue) { sqs_queue } + allow(sqs_msg).to receive(:message_id).and_return('test-message-id') + allow(sqs_msg).to receive(:change_visibility).and_raise(StandardError, 'AWS error') + + expect(Shoryuken.logger).to receive(:error) do |&block| + msg = block.call + expect(msg).to include('Could not auto extend the message') + expect(msg).to include('test-message-id') + expect(msg).to include('AWS error') + end + + Runner.new.run_and_sleep(TestWorker.new, queue, sqs_msg, visibility_timeout) + end + end + + context 'debug logging' do + it 'logs debug message when extending visibility' do + TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true + + allow(sqs_msg).to receive(:queue) { sqs_queue } + allow(sqs_msg).to receive(:message_id).and_return('test-message-id') + allow(sqs_msg).to receive(:change_visibility) + + expect(Shoryuken.logger).to receive(:debug) do |&block| + msg = block.call + expect(msg).to include('Extending message') + expect(msg).to include('test-message-id') + expect(msg).to include("by #{visibility_timeout}s") + end + + Runner.new.run_and_sleep(TestWorker.new, queue, sqs_msg, visibility_timeout) + end + end end diff --git a/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb b/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb similarity index 100% rename from spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb rename to spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb diff --git a/spec/shoryuken/middleware/server/timing_spec.rb b/spec/lib/shoryuken/middleware/server/timing_spec.rb similarity index 100% rename from spec/shoryuken/middleware/server/timing_spec.rb rename to spec/lib/shoryuken/middleware/server/timing_spec.rb diff --git a/spec/shoryuken/options_spec.rb b/spec/lib/shoryuken/options_spec.rb similarity index 100% rename from spec/shoryuken/options_spec.rb rename to spec/lib/shoryuken/options_spec.rb diff --git a/spec/shoryuken/polling/base_strategy_spec.rb b/spec/lib/shoryuken/polling/base_strategy_spec.rb similarity index 100% rename from spec/shoryuken/polling/base_strategy_spec.rb rename to spec/lib/shoryuken/polling/base_strategy_spec.rb diff --git a/spec/shoryuken/polling/queue_configuration_spec.rb b/spec/lib/shoryuken/polling/queue_configuration_spec.rb similarity index 100% rename from spec/shoryuken/polling/queue_configuration_spec.rb rename to spec/lib/shoryuken/polling/queue_configuration_spec.rb diff --git a/spec/shoryuken/polling/strict_priority_spec.rb b/spec/lib/shoryuken/polling/strict_priority_spec.rb similarity index 100% rename from spec/shoryuken/polling/strict_priority_spec.rb rename to spec/lib/shoryuken/polling/strict_priority_spec.rb diff --git a/spec/shoryuken/polling/weighted_round_robin_spec.rb b/spec/lib/shoryuken/polling/weighted_round_robin_spec.rb similarity index 100% rename from spec/shoryuken/polling/weighted_round_robin_spec.rb rename to spec/lib/shoryuken/polling/weighted_round_robin_spec.rb diff --git a/spec/shoryuken/processor_spec.rb b/spec/lib/shoryuken/processor_spec.rb similarity index 100% rename from spec/shoryuken/processor_spec.rb rename to spec/lib/shoryuken/processor_spec.rb diff --git a/spec/shoryuken/queue_spec.rb b/spec/lib/shoryuken/queue_spec.rb similarity index 100% rename from spec/shoryuken/queue_spec.rb rename to spec/lib/shoryuken/queue_spec.rb diff --git a/spec/shoryuken/runner_spec.rb b/spec/lib/shoryuken/runner_spec.rb similarity index 100% rename from spec/shoryuken/runner_spec.rb rename to spec/lib/shoryuken/runner_spec.rb diff --git a/spec/shoryuken/util_spec.rb b/spec/lib/shoryuken/util_spec.rb similarity index 95% rename from spec/shoryuken/util_spec.rb rename to spec/lib/shoryuken/util_spec.rb index 274695c2..0a081e25 100644 --- a/spec/shoryuken/util_spec.rb +++ b/spec/lib/shoryuken/util_spec.rb @@ -26,7 +26,7 @@ end let(:message_attributes) do - { 'shoryuken_class' => { string_value: ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper.to_s } } + { 'shoryuken_class' => { string_value: 'Shoryuken::ActiveJob::JobWrapper' } } end let(:body) do diff --git a/spec/lib/shoryuken/version_spec.rb b/spec/lib/shoryuken/version_spec.rb new file mode 100644 index 00000000..52023910 --- /dev/null +++ b/spec/lib/shoryuken/version_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Shoryuken::VERSION do + it 'has a version number' do + expect(Shoryuken::VERSION).not_to be_nil + end + + it 'follows semantic versioning format' do + expect(Shoryuken::VERSION).to match(/^\d+\.\d+\.\d+/) + end + + it 'is a string' do + expect(Shoryuken::VERSION).to be_a(String) + end +end \ No newline at end of file diff --git a/spec/shoryuken/worker/default_executor_spec.rb b/spec/lib/shoryuken/worker/default_executor_spec.rb similarity index 100% rename from spec/shoryuken/worker/default_executor_spec.rb rename to spec/lib/shoryuken/worker/default_executor_spec.rb diff --git a/spec/shoryuken/worker/inline_executor_spec.rb b/spec/lib/shoryuken/worker/inline_executor_spec.rb similarity index 100% rename from spec/shoryuken/worker/inline_executor_spec.rb rename to spec/lib/shoryuken/worker/inline_executor_spec.rb diff --git a/spec/lib/shoryuken/worker_registry_spec.rb b/spec/lib/shoryuken/worker_registry_spec.rb new file mode 100644 index 00000000..bef75d3f --- /dev/null +++ b/spec/lib/shoryuken/worker_registry_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Shoryuken::WorkerRegistry do + subject { described_class.new } + + describe '#batch_receive_messages?' do + it 'raises NotImplementedError' do + expect { subject.batch_receive_messages?('test-queue') }.to raise_error(NotImplementedError) + end + end + + describe '#clear' do + it 'raises NotImplementedError' do + expect { subject.clear }.to raise_error(NotImplementedError) + end + end + + describe '#fetch_worker' do + it 'raises NotImplementedError' do + queue = 'test-queue' + message = double('message') + expect { subject.fetch_worker(queue, message) }.to raise_error(NotImplementedError) + end + end + + describe '#queues' do + it 'raises NotImplementedError' do + expect { subject.queues }.to raise_error(NotImplementedError) + end + end + + describe '#register_worker' do + it 'raises NotImplementedError' do + queue = 'test-queue' + worker_class = Class.new + expect { subject.register_worker(queue, worker_class) }.to raise_error(NotImplementedError) + end + end + + describe '#workers' do + it 'raises NotImplementedError' do + expect { subject.workers('test-queue') }.to raise_error(NotImplementedError) + end + end + + context 'interface documentation' do + it 'defines the required interface methods' do + expect(subject).to respond_to(:batch_receive_messages?) + expect(subject).to respond_to(:clear) + expect(subject).to respond_to(:fetch_worker) + expect(subject).to respond_to(:queues) + expect(subject).to respond_to(:register_worker) + expect(subject).to respond_to(:workers) + end + + it 'is designed to be subclassed' do + expect(described_class).to be < Object + expect(described_class.ancestors).to include(described_class) + end + end +end \ No newline at end of file diff --git a/spec/shoryuken/worker_spec.rb b/spec/lib/shoryuken/worker_spec.rb similarity index 100% rename from spec/shoryuken/worker_spec.rb rename to spec/lib/shoryuken/worker_spec.rb diff --git a/spec/shoryuken_spec.rb b/spec/lib/shoryuken_spec.rb similarity index 100% rename from spec/shoryuken_spec.rb rename to spec/lib/shoryuken_spec.rb diff --git a/spec/shared_examples_for_active_job.rb b/spec/shared_examples_for_active_job.rb index 53f40f1f..c7c0ba56 100644 --- a/spec/shared_examples_for_active_job.rb +++ b/spec/shared_examples_for_active_job.rb @@ -1,5 +1,5 @@ require 'active_job' -require 'shoryuken/extensions/active_job_extensions' +require 'active_job/extensions' # Stand-in for a job class specified by the user class TestJob < ActiveJob::Base; end @@ -23,11 +23,11 @@ class TestJob < ActiveJob::Base; end specify do expect(queue).to receive(:send_message) do |hash| expect(hash[:message_deduplication_id]).to_not be - expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s) + expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(Shoryuken::ActiveJob::JobWrapper.to_s) expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq('String') expect(hash[:message_attributes].keys).to eq(['shoryuken_class']) end - expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper) + expect(Shoryuken).to receive(:register_worker).with(job.queue_name, Shoryuken::ActiveJob::JobWrapper) subject.enqueue(job) end @@ -50,7 +50,7 @@ class TestJob < ActiveJob::Base; end expect(hash[:message_deduplication_id]).to eq(message_deduplication_id) end - expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper) + expect(Shoryuken).to receive(:register_worker).with(job.queue_name, Shoryuken::ActiveJob::JobWrapper) subject.enqueue(job) end @@ -132,12 +132,12 @@ class TestJob < ActiveJob::Base; end } expect(queue).to receive(:send_message) do |hash| - expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s) + expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(Shoryuken::ActiveJob::JobWrapper.to_s) expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq('String') expect(hash[:message_attributes]['tracer_id'][:string_value]).to eq(custom_message_attributes['tracer_id'][:string_value]) expect(hash[:message_attributes]['tracer_id'][:data_type]).to eq('String') end - expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper) + expect(Shoryuken).to receive(:register_worker).with(job.queue_name, Shoryuken::ActiveJob::JobWrapper) subject.enqueue(job, message_attributes: custom_message_attributes) end @@ -158,7 +158,7 @@ class TestJob < ActiveJob::Base; end expect(queue).to receive(:send_message) do |hash| expect(hash[:message_attributes]['tracer_id']).to eq({ data_type: 'String', string_value: 'job-value' }) expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', - string_value: described_class::JobWrapper.to_s }) + string_value: Shoryuken::ActiveJob::JobWrapper.to_s }) end subject.enqueue job end @@ -189,7 +189,7 @@ class TestJob < ActiveJob::Base; end expect(hash[:message_attributes]['options_tracer_id']).to eq({ data_type: 'String', string_value: 'options-value' }) expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', - string_value: described_class::JobWrapper.to_s }) + string_value: Shoryuken::ActiveJob::JobWrapper.to_s }) end subject.enqueue job, message_attributes: custom_message_attributes end @@ -274,7 +274,7 @@ class TestJob < ActiveJob::Base; end expect(hash[:delay_seconds]).to eq(delay) end - expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper) + expect(Shoryuken).to receive(:register_worker).with(job.queue_name, Shoryuken::ActiveJob::JobWrapper) # need to figure out what to require Time.current and N.minutes to remove the stub allow(subject).to receive(:calculate_delay).and_return(delay) diff --git a/spec/shoryuken/extensions/active_job_adapter_spec.rb b/spec/shoryuken/extensions/active_job_adapter_spec.rb deleted file mode 100644 index 5c47f455..00000000 --- a/spec/shoryuken/extensions/active_job_adapter_spec.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -require 'shared_examples_for_active_job' -require 'shoryuken/extensions/active_job_adapter' - -RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do - include_examples 'active_job_adapters' -end diff --git a/spec/shoryuken/extensions/active_job_base_spec.rb b/spec/shoryuken/extensions/active_job_base_spec.rb deleted file mode 100644 index 162cb687..00000000 --- a/spec/shoryuken/extensions/active_job_base_spec.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -require 'active_job' -require 'shoryuken/extensions/active_job_extensions' -require 'shoryuken/extensions/active_job_adapter' - -RSpec.describe ActiveJob::Base do - let(:queue_adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new } - - subject do - worker_class = Class.new(described_class) - Object.const_set :MyWorker, worker_class - worker_class.queue_adapter = queue_adapter - worker_class - end - - after do - Object.send :remove_const, :MyWorker - end - - describe '#perform_now' do - it 'allows keyword args' do - collaborator = double 'worker collaborator' - subject.send(:define_method, :perform) do |**kwargs| - collaborator.foo(**kwargs) - end - expect(collaborator).to receive(:foo).with(foo: 'bar') - subject.perform_now foo: 'bar' - end - end - - describe '#perform_later' do - it 'calls enqueue on the adapter with the expected job' do - expect(queue_adapter).to receive(:enqueue) do |job| - expect(job.arguments).to eq([1, 2]) - end - - subject.perform_later 1, 2 - end - - it 'passes message_group_id to the queue_adapter' do - expect(queue_adapter).to receive(:enqueue) do |job| - expect(job.sqs_send_message_parameters[:message_group_id]).to eq('group-2') - end - - subject.set(message_group_id: 'group-2').perform_later 1, 2 - end - - it 'passes message_deduplication_id to the queue_adapter' do - expect(queue_adapter).to receive(:enqueue) do |job| - expect(job.sqs_send_message_parameters[:message_deduplication_id]).to eq('dedupe-id') - end - - subject.set(message_deduplication_id: 'dedupe-id').perform_later 1, 2 - end - - it 'passes message_attributes to the queue_adapter' do - message_attributes = { - 'custom_tracing_id' => { - string_value: 'value', - data_type: 'String' - } - } - expect(queue_adapter).to receive(:enqueue) do |job| - expect(job.sqs_send_message_parameters[:message_attributes]).to eq(message_attributes) - end - - subject.set(message_attributes: message_attributes).perform_later 1, 2 - end - - it 'passes message_system_attributes to the queue_adapter' do - message_system_attributes = { - 'AWSTraceHeader' => { - string_value: 'trace_id', - data_type: 'String' - } - } - expect(queue_adapter).to receive(:enqueue) do |job| - expect(job.sqs_send_message_parameters[:message_system_attributes]).to eq(message_system_attributes) - end - - subject.set(message_system_attributes: message_system_attributes).perform_later 1, 2 - end - end -end diff --git a/spec/shoryuken/extensions/active_job_continuation_spec.rb b/spec/shoryuken/extensions/active_job_continuation_spec.rb deleted file mode 100644 index 61f0f1f8..00000000 --- a/spec/shoryuken/extensions/active_job_continuation_spec.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -require 'active_job' -require 'shared_examples_for_active_job' -require 'shoryuken/extensions/active_job_adapter' -require 'shoryuken/extensions/active_job_extensions' - -RSpec.describe 'ActiveJob Continuation support' do - let(:adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new } - let(:job) do - job = TestJob.new - job.sqs_send_message_parameters = {} - job - end - let(:queue) { double('Queue', fifo?: false) } - - before do - allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue) - allow(Shoryuken).to receive(:register_worker) - end - - describe '#stopping?' do - context 'when Launcher is not initialized' do - it 'returns false' do - runner = instance_double(Shoryuken::Runner, launcher: nil) - allow(Shoryuken::Runner).to receive(:instance).and_return(runner) - - expect(adapter.stopping?).to be false - end - end - - context 'when Launcher is initialized' do - let(:runner) { instance_double(Shoryuken::Runner) } - let(:launcher) { instance_double(Shoryuken::Launcher) } - - before do - allow(Shoryuken::Runner).to receive(:instance).and_return(runner) - allow(runner).to receive(:launcher).and_return(launcher) - end - - it 'returns false when not stopping' do - allow(launcher).to receive(:stopping?).and_return(false) - expect(adapter.stopping?).to be false - end - - it 'returns true when stopping' do - allow(launcher).to receive(:stopping?).and_return(true) - expect(adapter.stopping?).to be true - end - end - end - - describe '#enqueue_at with past timestamps' do - let(:past_timestamp) { Time.current.to_f - 60 } # 60 seconds ago - - it 'enqueues with negative delay_seconds when timestamp is in the past' do - expect(queue).to receive(:send_message) do |hash| - expect(hash[:delay_seconds]).to be <= 0 - expect(hash[:delay_seconds]).to be >= -61 # Allow for rounding and timing - end - - adapter.enqueue_at(job, past_timestamp) - end - - it 'does not raise an error for past timestamps' do - allow(queue).to receive(:send_message) - - expect { adapter.enqueue_at(job, past_timestamp) }.not_to raise_error - end - end - - describe '#enqueue_at with future timestamps' do - let(:future_timestamp) { Time.current.to_f + 60 } # 60 seconds from now - - it 'enqueues with delay_seconds when timestamp is in the future' do - expect(queue).to receive(:send_message) do |hash| - expect(hash[:delay_seconds]).to be > 0 - expect(hash[:delay_seconds]).to be <= 60 - end - - adapter.enqueue_at(job, future_timestamp) - end - end - - describe '#enqueue_at with current timestamp' do - let(:current_timestamp) { Time.current.to_f } - - it 'enqueues with delay_seconds close to 0' do - expect(queue).to receive(:send_message) do |hash| - expect(hash[:delay_seconds]).to be_between(-1, 1) # Allow for timing/rounding - end - - adapter.enqueue_at(job, current_timestamp) - end - end - - describe 'retry_on with zero wait' do - it 'allows immediate retries through continuation mechanism' do - # Simulate a job with retry_on configuration that uses zero wait - past_timestamp = Time.current.to_f - 1 - - expect(queue).to receive(:send_message) do |hash| - # Negative delay for past timestamp - SQS will handle immediate delivery - expect(hash[:delay_seconds]).to be <= 0 - end - - adapter.enqueue_at(job, past_timestamp) - end - end -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 280ba152..e784d5fc 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -7,6 +7,7 @@ require 'warning' Warning.process do |warning| + next next unless warning.include?(Dir.pwd) next if warning.include?('useless use of a variable in void context') && warning.include?('core_ext') next if warning.include?('vendor/') @@ -29,8 +30,28 @@ require 'ostruct' Dotenv.load -require 'simplecov' -SimpleCov.start +unless ENV['SIMPLECOV_DISABLED'] + require 'simplecov' + SimpleCov.start do + add_filter '/spec/' + add_filter '/test_workers/' + add_filter '/examples/' + add_filter '/vendor/' + add_filter '/.bundle/' + + add_group 'Library', 'lib/' + add_group 'ActiveJob', 'lib/active_job' + add_group 'Middleware', 'lib/shoryuken/middleware' + add_group 'Polling', 'lib/shoryuken/polling' + add_group 'Workers', 'lib/shoryuken/worker' + add_group 'Helpers', 'lib/shoryuken/helpers' + + enable_coverage :branch + + minimum_coverage 89 + minimum_coverage_by_file 60 + end +end config_file = File.join(File.expand_path('..', __dir__), 'spec', 'shoryuken.yml')