Skip to content

Commit 3182ccd

Browse files
adamwclaude
andauthored
Fix flaky CircuitBreakerTest timing assertions (#452)
## Problem CI run [26810641361](https://github.com/softwaremill/ox/actions/runs/26810641361/job/79039384686) failed on the `ci (21)` matrix entry in `ox.resilience.CircuitBreakerTest` (`CircuitBreakerTest.scala:153`, test "should switch back to open after configured timeout in half open state"). ## Root cause The scheduled circuit-breaker state transitions (open → half-open → open) are driven by forked timers handed off to the breaker's actor. The exact moment a transition becomes observable depends on thread scheduling, so on a loaded CI runner a transition can land later than its nominal delay. The tests asserted state with a single fixed `sleep`-then-read, which is racy — hence the intermittent failure. ## Fix Replace the fixed-`sleep`-then-read assertions with ScalaTest `Eventually` polling (8s timeout, 50ms interval) for the expected state. This keeps the assertions deterministic in intent while tolerating scheduling jitter. No production code changed. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 4969867 commit 3182ccd

1 file changed

Lines changed: 25 additions & 14 deletions

File tree

core/src/test/scala/ox/resilience/CircuitBreakerTest.scala

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,14 @@ import org.scalatest.OptionValues
66
import scala.concurrent.duration.*
77
import ox.*
88
import org.scalatest.EitherValues
9+
import org.scalatest.concurrent.Eventually
10+
import org.scalatest.time.{Millis, Seconds, Span}
11+
12+
class CircuitBreakerTest extends AnyFlatSpec with Matchers with OptionValues with EitherValues with Eventually:
13+
// Scheduled state transitions are timer-driven, so their observable timing varies with thread scheduling - a single
14+
// timed read is racy (it caused an intermittent CI failure). The scheduled-state-change tests below poll with
15+
// `eventually` instead, each with a timeout/interval sized to its own config.
916

10-
class CircuitBreakerTest extends AnyFlatSpec with Matchers with OptionValues with EitherValues:
1117
behavior of "Circuit Breaker run operations"
1218

1319
it should "run operation when metrics are not exceeded" in supervised {
@@ -112,15 +118,17 @@ class CircuitBreakerTest extends AnyFlatSpec with Matchers with OptionValues wit
112118

113119
// when
114120
val result1 = circuitBreaker.runOrDropEither(f())
115-
sleep(100.millis) // wait for state to register
116-
val state = circuitBreaker.stateMachine.state
117-
sleep(1500.millis)
118-
val stateAfterWait = circuitBreaker.stateMachine.state
119121

120122
// then
121123
result1 shouldBe defined
122-
state shouldBe a[CircuitBreakerState.Open]
123-
stateAfterWait shouldBe a[CircuitBreakerState.HalfOpen]
124+
// the failing call opens the breaker (immediate)
125+
eventually(timeout(Span(2, Seconds)), interval(Span(50, Millis))) {
126+
circuitBreaker.stateMachine.state shouldBe a[CircuitBreakerState.Open]
127+
}
128+
// after waitDurationOpenState (1s) it switches to half-open, and stays (halfOpenTimeoutDuration defaults to 0)
129+
eventually(timeout(Span(5, Seconds)), interval(Span(50, Millis))) {
130+
circuitBreaker.stateMachine.state shouldBe a[CircuitBreakerState.HalfOpen]
131+
}
124132
}
125133

126134
it should "switch back to open after configured timeout in half open state" in supervised {
@@ -134,23 +142,26 @@ class CircuitBreakerTest extends AnyFlatSpec with Matchers with OptionValues wit
134142
slidingWindow = SlidingWindow.CountBased(numberOfOperations),
135143
numberOfCallsInHalfOpenState = 1,
136144
waitDurationOpenState = 1.seconds,
137-
halfOpenTimeoutDuration = 2.seconds
145+
// wide enough that the transient half-open state is reliably sampled by the 50ms poll below
146+
halfOpenTimeoutDuration = 4.seconds
138147
)
139148
)
140149
def f(): Either[String, String] =
141150
Left("boom")
142151

143152
// when
144153
val result1 = circuitBreaker.runOrDropEither(f()) // trigger switch to open
145-
sleep(1500.millis) // wait for state to register, and for switch to half open
146-
val state = circuitBreaker.stateMachine.state
147-
sleep(2500.millis) // wait longer than half open timeout
148-
val stateAfterWait = circuitBreaker.stateMachine.state
149154

150155
// then
151156
result1 shouldBe defined
152-
state shouldBe a[CircuitBreakerState.HalfOpen]
153-
stateAfterWait shouldBe a[CircuitBreakerState.Open]
157+
// switches to half-open after 1s; timeout sits inside the 4s half-open window to catch this transient state
158+
eventually(timeout(Span(4, Seconds)), interval(Span(50, Millis))) {
159+
circuitBreaker.stateMachine.state shouldBe a[CircuitBreakerState.HalfOpen]
160+
}
161+
// no half-open call completes, so it switches back to open; timeout outlasts the 4s half-open window
162+
eventually(timeout(Span(8, Seconds)), interval(Span(50, Millis))) {
163+
circuitBreaker.stateMachine.state shouldBe a[CircuitBreakerState.Open]
164+
}
154165
}
155166

156167
it should "correctly transitions through states when there are concurrently running operations" in supervised {

0 commit comments

Comments
 (0)