Improve reaper performance under heavy load (#666)

* Improve reaper performance under heavy load

Closes #663

Signed-off-by: mhenrixon <[email protected]>

* Mandatory linter commit

.reek.yml

@@ -42,6 +42,7 @@ detectors:
       - SidekiqUniqueJobs::Locksmith#wait_for_primed_token
       - SidekiqUniqueJobs::OnConflict::Reschedule#call
       - SidekiqUniqueJobs::Orphans::RubyReaper#entries
+      - SidekiqUniqueJobs::Orphans::RubyReaper#orphans
       - SidekiqUniqueJobs::Profiler#self.stop
       - SidekiqUniqueJobs::Script::Caller#extract_args
       - SidekiqUniqueJobs::Timing#timed
@@ -147,6 +148,7 @@ detectors:
       - SidekiqUniqueJobs::Orphans::RubyReaper#active?
       - SidekiqUniqueJobs::Orphans::RubyReaper#enqueued?
       - SidekiqUniqueJobs::Orphans::RubyReaper#entries
+      - SidekiqUniqueJobs::Orphans::RubyReaper#orphans
       - SidekiqUniqueJobs::Profiler#self.stop
       - SidekiqUniqueJobs::Script::Caller#call_script
       - SidekiqUniqueJobs::Script::Caller#extract_args

lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -9,6 +9,7 @@ module Orphans
     #
     # @author Mikael Henriksson <[email protected]>
     #
+    # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
       #
       # @return [String] the suffix for :RUN locks
@@ -54,13 +55,27 @@ def call
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans
-        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
-          next if belongs_to_job?(digest)
+      def orphans # rubocop:disable Metrics/MethodLength
+        page = 0
+        per = reaper_count * 2
+        orphans = []
+        results = conn.zrange(digests.key, page * per, (page + 1) * per)
 
-          memo << digest
-          break memo if memo.size >= reaper_count
+        while results.size.positive?
+          results.each do |digest|
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
+
+          break if orphans.size >= reaper_count
+
+          page += 1
+          results = conn.zrange(digests.key, page * per, (page + 1) * per)
         end
+
+        orphans
       end
 
       #
@@ -211,5 +226,6 @@ def in_sorted_set?(key, digest)
         conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

spec/sidekiq_unique_jobs/orphans/ruby_reaper_spec.rb

@@ -49,7 +49,8 @@
       it "returns the first digest" do
         SidekiqUniqueJobs.use_config(reaper_count: 1) do
           expect(orphans.size).to eq(1)
-          expect([digest_one, digest_two, digest_three]).to include(orphans.first)
+          # This is the first one to be created and should therefor be the only match
+          expect(orphans).to match_array([digest])
         end
       end
     end