From f1a192e463ed9e7d6e3d50d6ee7045b92e9dad2b Mon Sep 17 00:00:00 2001 From: Robert Gradowski Date: Thu, 24 Oct 2024 14:52:10 +0200 Subject: [PATCH] another spec version --- .gitignore | 1 + .rubocop.yml | 114 +++++++++++++++++++++++- Gemfile | 20 +++-- Gemfile.lock | 94 ------------------- README.md | 5 ++ Rakefile | 4 +- lib/jobs/my_job.rb | 12 +++ lib/sidekiq-poison-pill-remedy.rb | 2 +- lib/sidekiq_poison_pill_remedy.rb | 6 +- lib/sidekiq_worker_loader.rb | 8 ++ lib/version.rb | 2 +- sidekiq-poison-pill-remedy.gemspec | 29 +++--- spec/sidekiq_poison_pill_remedy_spec.rb | 47 +++++----- spec/spec_helper.rb | 18 ++-- spec/support/my_job.rb | 11 +-- 15 files changed, 208 insertions(+), 165 deletions(-) delete mode 100644 Gemfile.lock create mode 100644 lib/jobs/my_job.rb create mode 100644 lib/sidekiq_worker_loader.rb diff --git a/.gitignore b/.gitignore index 541570a..1c05247 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ # rspec failure tracking .rspec_status +# Ignore Gemfile.lock Gemfile.lock \ No newline at end of file diff --git a/.rubocop.yml b/.rubocop.yml index 2b9e324..6aee70e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,8 +1,114 @@ -Style/Documentation: - Enabled: false +require: + - rubocop-performance + - rubocop-rspec + +inherit_mode: + merge: + - Exclude + AllCops: NewCops: enable + TargetRubyVersion: 3.1 + Exclude: + - "db/**/*" + - "bin/**/*" + - "tmp/**/*" + - "log/**/*" + - "vendor/**/*" + - "spec/rails_helper.rb" + +Layout/FrozenStringLiteralComment: + Enabled: true + +Layout/ArgumentAlignment: + EnforcedStyle: with_fixed_indentation + +Layout/ArrayAlignment: + EnforcedStyle: with_fixed_indentation + +Layout/EmptyLineBetweenDefs: + AllowAdjacentOneLineDefs: true + +Layout/EndAlignment: + EnforcedStyleAlignWith: variable + +Layout/FirstArgumentIndentation: + EnforcedStyle: consistent + +Layout/FirstArrayElementIndentation: + EnforcedStyle: consistent + +Layout/FirstHashElementIndentation: + EnforcedStyle: consistent + +Layout/MultilineMethodCallIndentation: + EnforcedStyle: indented + +Layout/MultilineOperationIndentation: + EnforcedStyle: indented + +Layout/ParameterAlignment: + EnforcedStyle: with_fixed_indentation + +Layout/SpaceBeforeBrackets: + Enabled: false + + +Lint/UnusedMethodArgument: + AllowUnusedKeywordArguments: true + +Lint/AmbiguousBlockAssociation: + Exclude: + - 'spec/**/*' + + +Naming/FileName: + EnforcedStyle: snake_case + Exclude: + - 'lib/sidekiq-poison-pill-remedy.rb' + +Naming/VariableNumber: + EnforcedStyle: snake_case + + +RSpec/MultipleExpectations: + Max: 20 + +RSpec/NestedGroups: + Max: 10 + +RSpec/ExampleLength: + Max: 15 + Exclude: + - 'spec/sidekiq_poison_pill_remedy_spec.rb' + +RSpec/VerifiedDoubles: + Exclude: + - 'spec/**/*.rb' + + Metrics/AbcSize: - Max: 19 + Max: 15 + Exclude: + - 'lib/sidekiq_poison_pill_remedy.rb' + + Metrics/MethodLength: - Max: 20 \ No newline at end of file + Max: 20 + +Metrics/BlockLength: + Exclude: + - 'lib/sidekiq_poison_pill_remedy.rb' + +Layout/LineLength: + Max: 125 + + +Style/StringLiterals: + EnforcedStyle: double_quotes + ConsistentQuotesInMultiline: true + +Style/Documentation: + Enabled: false + + diff --git a/Gemfile b/Gemfile index 7d28ee2..9b5a7c8 100644 --- a/Gemfile +++ b/Gemfile @@ -1,14 +1,18 @@ # frozen_string_literal: true -source 'https://rubygems.org' +source "https://rubygems.org" # Specify your gem's dependencies in sidekiq-poison-pill-remedy.gemspec gemspec -gem 'rake', '~> 13.0' -gem 'rspec', '~> 3.0' -gem 'rubocop', '~> 1.40', require: false -gem 'rubocop-performance' -gem 'rubocop-rake' -gem 'rubocop-rspec' -gem 'sentry-ruby' +gem "rake", "~> 13.0" +gem "sentry-ruby" + +group :development, :test do + gem "rspec", "~> 3.0" + gem "rspec-sidekiq" + gem "rubocop", "~> 1.40", require: false + gem "rubocop-performance" + gem "rubocop-rake" + gem "rubocop-rspec" +end diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index abcb94b..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,94 +0,0 @@ -PATH - remote: . - specs: - sidekiq-poison-pill-remedy (0.1.0) - sidekiq - -GEM - remote: https://rubygems.org/ - specs: - ast (2.4.2) - bigdecimal (3.1.8) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) - diff-lcs (1.5.1) - json (2.7.2) - language_server-protocol (3.17.0.3) - logger (1.6.1) - parallel (1.26.3) - parser (3.3.5.0) - ast (~> 2.4.1) - racc - racc (1.8.1) - rack (3.1.7) - rainbow (3.1.1) - rake (13.2.1) - redis-client (0.22.2) - connection_pool - regexp_parser (2.9.2) - rspec (3.13.0) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.1) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-mocks (3.13.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-sidekiq (5.0.0) - rspec-core (~> 3.0) - rspec-expectations (~> 3.0) - rspec-mocks (~> 3.0) - sidekiq (>= 5, < 8) - rspec-support (3.13.1) - rubocop (1.66.1) - json (~> 2.3) - language_server-protocol (>= 3.17.0) - parallel (~> 1.10) - parser (>= 3.3.0.2) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.4, < 3.0) - rubocop-ast (>= 1.32.2, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.32.3) - parser (>= 3.3.1.0) - rubocop-performance (1.22.1) - rubocop (>= 1.48.1, < 2.0) - rubocop-ast (>= 1.31.1, < 2.0) - rubocop-rake (0.6.0) - rubocop (~> 1.0) - rubocop-rspec (3.1.0) - rubocop (~> 1.61) - ruby-progressbar (1.13.0) - sentry-ruby (5.21.0) - bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) - sidekiq (7.3.2) - concurrent-ruby (< 2) - connection_pool (>= 2.3.0) - logger - rack (>= 2.2.4) - redis-client (>= 0.22.2) - unicode-display_width (2.6.0) - -PLATFORMS - arm64-darwin-22 - x86_64-linux - -DEPENDENCIES - rake (~> 13.0) - rspec (~> 3.0) - rspec-sidekiq - rubocop (~> 1.40) - rubocop-performance - rubocop-rake - rubocop-rspec - sentry-ruby - sidekiq-poison-pill-remedy! - -BUNDLED WITH - 2.4.18 diff --git a/README.md b/README.md index fb45eab..a917967 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,13 @@ Or install it yourself as: The gem is supposed to be used in the following way when added to the application +Check Sidekiq super_fetch:[here](https://github.com/sidekiq/sidekiq/wiki/Reliability#using-super_fetch) + +Remedy is supposed to be use like: `config.super_fetch!(&SidekiqPoisonPillRemedy.remedy)` +When a job fails, the SidekiqPoisonPillRemedy captures the failure and determines whether the job should be moved to a dedicated poison_pill queue. + ## Development After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. diff --git a/Rakefile b/Rakefile index 82bb534..b6ae734 100644 --- a/Rakefile +++ b/Rakefile @@ -1,7 +1,7 @@ # frozen_string_literal: true -require 'bundler/gem_tasks' -require 'rspec/core/rake_task' +require "bundler/gem_tasks" +require "rspec/core/rake_task" RSpec::Core::RakeTask.new(:spec) diff --git a/lib/jobs/my_job.rb b/lib/jobs/my_job.rb new file mode 100644 index 0000000..9d95ee1 --- /dev/null +++ b/lib/jobs/my_job.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "sidekiq" + +class MyJob + include Sidekiq::Job + sidekiq_options retry: 1 + + def perform(arg) + raise StandardError, "Forced failure for testing" if arg == "fail" + end +end diff --git a/lib/sidekiq-poison-pill-remedy.rb b/lib/sidekiq-poison-pill-remedy.rb index 78c0ba0..bd456f4 100644 --- a/lib/sidekiq-poison-pill-remedy.rb +++ b/lib/sidekiq-poison-pill-remedy.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -require 'sidekiq_poison_pill_remedy' +require "sidekiq_poison_pill_remedy" diff --git a/lib/sidekiq_poison_pill_remedy.rb b/lib/sidekiq_poison_pill_remedy.rb index 60e2331..bc74ca7 100644 --- a/lib/sidekiq_poison_pill_remedy.rb +++ b/lib/sidekiq_poison_pill_remedy.rb @@ -7,7 +7,7 @@ def self.remedy job = Sidekiq::DeadSet.new.find_job(pill.jid) - if job.queue == 'poison_pill' + if job.queue == "poison_pill" capture_sentry_message( "#{job.klass} failed in the `#{job.queue}`, this means that it has to be urgently optimized on memory usage", level: :critical, @@ -29,8 +29,8 @@ def self.capture_sentry_message(message, level:, job_item:) if defined?(Sentry) Sentry.capture_message( message, - level: level, - extra: { job_item: job_item } + level:, + extra: { job_item: } ) end diff --git a/lib/sidekiq_worker_loader.rb b/lib/sidekiq_worker_loader.rb new file mode 100644 index 0000000..3e8444b --- /dev/null +++ b/lib/sidekiq_worker_loader.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "sidekiq" +require "sidekiq/api" +require "sidekiq/testing" +require "rspec-sidekiq" +require "sidekiq-poison-pill-remedy" +require_relative "jobs/my_job" diff --git a/lib/version.rb b/lib/version.rb index 214cd96..921083c 100644 --- a/lib/version.rb +++ b/lib/version.rb @@ -3,5 +3,5 @@ module SidekiqPoisonPillRemedy module Version end - VERSION = '0.1.0' + VERSION = "0.1.0" end diff --git a/sidekiq-poison-pill-remedy.gemspec b/sidekiq-poison-pill-remedy.gemspec index 577a744..5896b3e 100644 --- a/sidekiq-poison-pill-remedy.gemspec +++ b/sidekiq-poison-pill-remedy.gemspec @@ -1,22 +1,22 @@ # frozen_string_literal: true -require_relative 'lib/version' +require_relative "lib/version" Gem::Specification.new do |spec| - spec.name = 'sidekiq-poison-pill-remedy' + spec.name = "sidekiq-poison-pill-remedy" spec.version = SidekiqPoisonPillRemedy::VERSION - spec.authors = ['Karol Galanciak'] - spec.email = ['karol.galanciak@gmail.com'] + spec.authors = ["Karol Galanciak"] + spec.email = ["karol.galanciak@gmail.com"] spec.summary = "Enhances Sidekiq's job processing by automatically handling and rescheduling poison pills" spec.description = "Enhances Sidekiq's job processing by automatically handling and rescheduling poison pills" - spec.homepage = 'https://github.com/BookingSync/sidekiq-poison-pill-remedy' - spec.license = 'MIT' - spec.required_ruby_version = '>= 3.0.0' + spec.homepage = "https://github.com/BookingSync/sidekiq-poison-pill-remedy" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.1.0" - spec.metadata['homepage_uri'] = spec.homepage - spec.metadata['source_code_uri'] = 'https://github.com/BookingSync/sidekiq-poison-pill-remedy' - spec.metadata['changelog_uri'] = 'https://github.com/BookingSync/sidekiq-poison-pill-remedy/blob/master/CHANGELOG.md' + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "https://github.com/BookingSync/sidekiq-poison-pill-remedy" + spec.metadata["changelog_uri"] = "https://github.com/BookingSync/sidekiq-poison-pill-remedy/blob/master/CHANGELOG.md" # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. @@ -27,16 +27,15 @@ Gem::Specification.new do |spec| f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile]) end end - spec.bindir = 'exe' + spec.bindir = "exe" spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } - spec.require_paths = ['lib'] + spec.require_paths = ["lib"] # Uncomment to register a new dependency of your gem # spec.add_dependency "example-gem", "~> 1.0" - spec.add_dependency 'sidekiq' - spec.add_development_dependency 'rspec-sidekiq' + spec.add_dependency "sidekiq" # For more information and examples about making a new gem, check out our # guide at: https://bundler.io/guides/creating_gem.html - spec.metadata['rubygems_mfa_required'] = 'true' + spec.metadata["rubygems_mfa_required"] = "true" end diff --git a/spec/sidekiq_poison_pill_remedy_spec.rb b/spec/sidekiq_poison_pill_remedy_spec.rb index 20cbef6..858bae5 100644 --- a/spec/sidekiq_poison_pill_remedy_spec.rb +++ b/spec/sidekiq_poison_pill_remedy_spec.rb @@ -1,41 +1,46 @@ # frozen_string_literal: true -require 'sidekiq/testing' -require 'rspec-sidekiq' -require 'sidekiq' -require 'sidekiq-poison-pill-remedy' -require 'support/my_job' -require 'sidekiq/api' +require "sidekiq" +require "sidekiq/api" +require "sidekiq/testing" +require "rspec-sidekiq" +require "sidekiq-poison-pill-remedy" +require "support/my_job" RSpec.describe SidekiqPoisonPillRemedy do before do - Sidekiq::Testing.fake! + Sidekiq::Testing.inline! + allow(Sentry).to receive(:capture_message) end - it 'moves job to poison_pill queue and logs message' do - puts '1' - jid = MyJob.perform_async('fail') + + it "moves job to poison_pill queue and logs message" do + puts "1" + + jid = nil + expect do + jid = MyJob.perform_async("fail") + end.to raise_error(StandardError, "Forced failure for testing") + puts "Jobs in Queue: #{Sidekiq::Queue.new.size}" puts "DeadSet size: #{Sidekiq::DeadSet.new.size}" - puts '2' - MyJob.drain - puts '3' + puts "Jobs in DeadSet after processing: #{Sidekiq::DeadSet.new.size}" - job = Sidekiq::DeadSet.new.find_job(jid) - puts '4' - puts "Job JID: #{jid}" - puts "DeadSet size: #{Sidekiq::DeadSet.new.size}" + dead_set = Sidekiq::DeadSet.new + expect(dead_set.size).to eq(1) + job = dead_set.find_job(jid) + puts "4" + puts "Job JID: #{jid}" expect(job).not_to be_nil - expect(job.klass).to eq('MyJob') + expect(job.klass).to eq("MyJob") - SidekiqPoisonPillRemedy.remedy.call(nil, job) + expect { described_class.remedy.call(nil, job) }.not_to raise_error puts "DeadSet jobs: #{Sidekiq::DeadSet.new.size}" puts "Queue jobs: #{Sidekiq::Queue.new.size}" - puts "Job JID: #{jid}" - poison_pill_job = Sidekiq::Queue.new('poison_pill').find_job(job.jid) + poison_pill_job = Sidekiq::Queue.new("poison_pill").find_job(job.jid) expect(poison_pill_job).not_to be_nil puts "Poison Pill Job JID: #{poison_pill_job.jid}" end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 439897b..fad5a51 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,18 +1,18 @@ # frozen_string_literal: true -require 'bundler/setup' -require 'sidekiq/testing' -require 'rspec-sidekiq' -require 'sentry-ruby' -require 'sidekiq' -require 'sidekiq-poison-pill-remedy' +require "bundler/setup" +require "sidekiq/testing" +require "rspec-sidekiq" +require "sentry-ruby" +require "sidekiq" +require "sidekiq-poison-pill-remedy" -Dir[File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f } +Dir[File.join(__dir__, "support", "**", "*.rb")].each { |f| require f } RSpec.configure do |config| Sidekiq::Testing.inline! # Enable flags like --only-failures and --next-failure - config.example_status_persistence_file_path = '.rspec_status' + config.example_status_persistence_file_path = ".rspec_status" # Disable RSpec exposing methods globally on `Module` and `main` config.disable_monkey_patching! @@ -22,6 +22,6 @@ end config.before(:all) do - ENV['REDIS_URL'] ||= 'redis://localhost:6379/1' + ENV["REDIS_URL"] ||= "redis://localhost:6379/1" end end diff --git a/spec/support/my_job.rb b/spec/support/my_job.rb index e8b2b9a..0137189 100644 --- a/spec/support/my_job.rb +++ b/spec/support/my_job.rb @@ -1,13 +1,10 @@ +# frozen_string_literal: true + class MyJob include Sidekiq::Job - sidekiq_options retry: 0 + sidekiq_options retry: 3 def perform(arg) - if arg == 'fail' - Sidekiq.logger.error('Forced failure for testing') - - else - 'surprise' - end + raise StandardError, "Forced failure for testing" if arg == "fail" end end