Fail incomplete worker-history retries

Author: dpashk-figma

ruby/lib/ci/queue/redis.rb

@@ -17,6 +17,7 @@ module CI
   module Queue
     module Redis
       Error = Class.new(StandardError)
+      IncompleteRetry = Class.new(Error)
       LostMaster = Class.new(Error)
       WorkerHistoryError = Class.new(Error)
 

ruby/lib/ci/queue/redis/retry.rb

@@ -3,18 +3,27 @@ module CI
   module Queue
     module Redis
       class Retry < Static
-        def initialize(tests, config, redis:)
+        def initialize(tests, config, redis:, require_exhaustion: false)
           @redis = redis
+          @require_exhaustion = require_exhaustion
           super(tests, config)
         end
 
         def build
           @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
+        def poll
+          super
+          return unless require_exhaustion
+          return if exhausted?
+
+          raise IncompleteRetry, 'Worker history replay stopped before completion'
+        end
+
         private
 
-        attr_reader :redis
+        attr_reader :redis, :require_exhaustion
       end
     end
   end

ruby/lib/ci/queue/redis/worker.rb

@@ -156,7 +156,12 @@ def retry_queue(selection: :failed_tests)
                        raise ArgumentError, "Unknown retry selection: #{selection.inspect}"
                      end
 
-          Retry.new(test_ids, config, redis: redis)
+          Retry.new(
+            test_ids,
+            config,
+            redis: redis,
+            require_exhaustion: selection == :worker_history
+          )
         end
 
         def supervisor

ruby/test/ci/queue/redis_test.rb

@@ -79,6 +79,20 @@ def test_worker_history_retry_requires_reservations
     assert_equal 'Reservation history is missing for worker 1', error.message
   end
 
+  def test_worker_history_retry_requires_complete_replay
+    poll(@queue)
+    retry_queue = populate(@queue.retry_queue(selection: :worker_history))
+
+    error = assert_raises(CI::Queue::Redis::IncompleteRetry) do
+      retry_queue.poll do
+        retry_queue.shutdown!
+      end
+    end
+
+    assert_equal 'Worker history replay stopped before completion', error.message
+    refute_predicate retry_queue, :exhausted?
+  end
+
   def test_retry_queue_rejects_unknown_selection
     error = assert_raises(ArgumentError) do
       @queue.retry_queue(selection: :everything)

ruby/test/integration/minitest_redis_test.rb

@@ -241,6 +241,23 @@ def test_worker_history_retry_fails_without_worker_reservations
       assert_includes out, 'Reservation history is missing for worker 2'
     end
 
+    def test_worker_history_retry_fails_when_replay_stops_early
+      run_worker_history_worker(
+        retry_count: 0,
+        build_id: 'incomplete-history',
+        test_file: 'test/failing_test.rb'
+      )
+      out, err = run_worker_history_worker(
+        retry_count: 1,
+        build_id: 'incomplete-history',
+        test_file: 'test/failing_test.rb',
+        extra_args: ['--max-consecutive-failures', '1']
+      )
+
+      refute_predicate $?, :success?
+      assert_includes out + err, 'Worker history replay stopped before completion'
+    end
+
     def test_retry_fails_when_test_run_is_expired
       out, err = capture_subprocess_io do
         system(
@@ -809,19 +826,29 @@ def test_utf8_tests_and_marshal
 
     private
 
-    def run_worker_history_worker(retry_count:, build_id: 'worker-history', worker_id: '1')
+    def run_worker_history_worker(
+      retry_count:,
+      build_id: 'worker-history',
+      worker_id: '1',
+      test_file: 'test/passing_test.rb',
+      extra_args: []
+    )
+      args = [
+        @exe, 'run',
+        '--queue', @redis_url,
+        '--seed', 'foobar',
+        '--build', build_id,
+        '--worker', worker_id,
+        '--timeout', '1',
+        '--retry-selection', 'worker-history'
+      ]
+      args.concat(extra_args)
+      args.push('-Itest', test_file)
+
       capture_subprocess_io do
         system(
           { 'BUILDKITE_RETRY_COUNT' => retry_count.to_s },
-          @exe, 'run',
-          '--queue', @redis_url,
-          '--seed', 'foobar',
-          '--build', build_id,
-          '--worker', worker_id,
-          '--timeout', '1',
-          '--retry-selection', 'worker-history',
-          '-Itest',
-          'test/passing_test.rb',
+          *args,
           chdir: 'test/fixtures/'
         )
       end