Skip to content

Enforce max reconnection attempts #5253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
karknu wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Enforce max reconnection attempts #5253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cardano-diffusion/changelog.d/20251128_104055_karl.fb.knutsson_max_recon_main.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Breaking

- Enforce a maximum failure limit on cold promotions

8 changes: 4 additions & 4 deletions cardano-diffusion/tests/lib/Test/Cardano/Network/Diffusion/Testnet.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2738,9 +2738,9 @@ prop_diffusion_target_established_local ioSimTrace traceNumber =
(Just . fromMaybe Set.empty)
. Signal.fromEvents
. Signal.selectEvents
(\case TracePromoteColdFailed _ _ peer _ _ ->
(\case TracePromoteColdFailed _ _ peer _ _ _ ->
Just (Set.singleton peer)
TracePromoteColdBigLedgerPeerFailed _ _ peer _ _ ->
TracePromoteColdBigLedgerPeerFailed _ _ peer _ _ _ ->
Just (Set.singleton peer)
TraceDemoteBigLedgerPeersAsynchronous status
| Set.null failures -> Nothing
Expand Down Expand Up @@ -3387,7 +3387,7 @@ prop_diffusion_async_demotions ioSimTrace traceNumber =
Just $ Stop failures
where
failures = Set.singleton peeraddr
TracePromoteColdFailed _ _ peeraddr _ _ ->
TracePromoteColdFailed _ _ peeraddr _ _ _ ->
Just $ Stop failures
where
failures = Set.singleton peeraddr
Expand All @@ -3399,7 +3399,7 @@ prop_diffusion_async_demotions ioSimTrace traceNumber =
Just $ Stop failures
where
failures = Set.singleton peeraddr
TracePromoteColdBigLedgerPeerFailed _ _ peeraddr _ _ ->
TracePromoteColdBigLedgerPeerFailed _ _ peeraddr _ _ _ ->
Just $ Stop failures
where
failures = Set.singleton peeraddr
Expand Down
11 changes: 6 additions & 5 deletions cardano-diffusion/tests/lib/Test/Cardano/Network/PeerSelection.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2650,7 +2650,7 @@ prop_governor_target_established_below (MaxTime maxTime) env =
(Just . fromMaybe Set.empty)
. Signal.fromEvents
. Signal.selectEvents
(\case TracePromoteColdFailed _ _ peer _ _ ->
(\case TracePromoteColdFailed _ _ peer _ _ _ ->
Just $! Set.singleton peer
TraceDemoteLocalAsynchronous status
| Set.null failures -> Nothing
Expand Down Expand Up @@ -2760,7 +2760,7 @@ prop_governor_target_established_big_ledger_peers_below (MaxTime maxTime) env =
(Just . fromMaybe Set.empty)
. Signal.fromEvents
. Signal.selectEvents
(\case TracePromoteColdBigLedgerPeerFailed _ _ peer _ _ ->
(\case TracePromoteColdBigLedgerPeerFailed _ _ peer _ _ _ ->
--TODO: the environment does not yet cause this to happen
-- it requires synchronous failure in the establish action
Just (Set.singleton peer)
Expand Down Expand Up @@ -3499,11 +3499,11 @@ prop_governor_target_established_local (MaxTime maxTime) env =
(Just . fromMaybe Set.empty)
. Signal.fromEvents
. Signal.selectEvents
(\case TracePromoteColdFailed _ _ peer _ _ ->
(\case TracePromoteColdFailed _ _ peer _ _ _ ->
--TODO: the environment does not yet cause this to happen
-- it requires synchronous failure in the establish action
Just (Set.singleton peer)
TracePromoteColdBigLedgerPeerFailed _ _ peer _ _ ->
TracePromoteColdBigLedgerPeerFailed _ _ peer _ _ _ ->
Just (Set.singleton peer)
TraceDemoteLocalAsynchronous status
| Set.null failures -> Nothing
Expand Down Expand Up @@ -4406,7 +4406,8 @@ _governorFindingPublicRoots targetNumberOfRootPeers readDomains readUseBootstrap
policyPeerShareRetryTime = 0, -- seconds
policyPeerShareBatchWaitTime = 0, -- seconds
policyPeerShareOverallTimeout = 0, -- seconds
policyPeerShareActivationDelay = 2 -- seconds
policyPeerShareActivationDelay = 2, -- seconds
policyMaxConnectionRetries = 5
}
pickTrivially :: Applicative m => Set SockAddr -> Int -> m (Set SockAddr)
pickTrivially m n = pure . Set.take n $ m
Expand Down
117 changes: 59 additions & 58 deletions cardano-diffusion/tests/lib/Test/Cardano/Network/PeerSelection/MockEnvironment.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -715,7 +715,8 @@ mockPeerSelectionPolicy GovernorMockEnvironment {
policyPeerShareRetryTime = 3600, -- seconds
policyPeerShareBatchWaitTime = 3, -- seconds
policyPeerShareOverallTimeout = 10, -- seconds
policyPeerShareActivationDelay = 300 -- seconds
policyPeerShareActivationDelay = 300, -- seconds
policyMaxConnectionRetries = 5
}

--
Expand Down Expand Up @@ -743,63 +744,63 @@ tracerTracePeerSelection = contramap f tracerTestTraceEvent
-- make the tracer strict
f :: TracePeerSelection extraState extraFlags extraPeers ExtraTrace PeerAddr
-> TestTraceEvent extraState extraFlags extraPeers extraCounters ExtraTrace
f a@(TraceLocalRootPeersChanged !_ !_) = GovernorEvent a
f a@(TraceTargetsChanged !_ !_) = GovernorEvent a
f a@(TracePublicRootsRequest !_ !_) = GovernorEvent a
f a@(TracePublicRootsResults !_ !_ !_) = GovernorEvent a
f a@(TracePublicRootsFailure !_ !_ !_) = GovernorEvent a
f a@(TraceForgetColdPeers !_ !_ !_) = GovernorEvent a
f a@(TraceBigLedgerPeersRequest !_ !_) = GovernorEvent a
f a@(TraceBigLedgerPeersResults !_ !_ !_) = GovernorEvent a
f a@(TraceBigLedgerPeersFailure !_ !_ !_) = GovernorEvent a
f a@(TraceForgetBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TracePickInboundPeers !_ !_ !_ !_) = GovernorEvent a
f a@(TracePeerShareRequests !_ !_ !_ !_ !_) = GovernorEvent a
f a@(TracePeerShareResults !_) = GovernorEvent a
f a@(TracePeerShareResultsFiltered !_) = GovernorEvent a
f a@(TracePromoteColdPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdLocalPeers !_ !_) = GovernorEvent a
f a@(TracePromoteColdFailed !_ !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdBigLedgerPeerFailed !_ !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmLocalPeers !_ !_) = GovernorEvent a
f a@(TracePromoteWarmFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmAborted !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeerFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeerAborted !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmBigLedgerPeerFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteLocalHotPeers !_ !_) = GovernorEvent a
f a@(TraceDemoteHotFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotBigLedgerPeerFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteAsynchronous !_) = GovernorEvent a
f a@(TraceDemoteLocalAsynchronous !_) = GovernorEvent a
f a@(TraceDemoteBigLedgerPeersAsynchronous !_) = GovernorEvent a
f a@TraceGovernorWakeup = GovernorEvent a
f a@(TraceChurnWait !_) = GovernorEvent a
f a@(ExtraTrace (TraceLedgerStateJudgementChanged !_)) = GovernorEvent a
f a@TraceOnlyBootstrapPeers = GovernorEvent a
f a@TraceBootstrapPeersFlagChangedWhilstInSensitiveState = GovernorEvent a
f a@(ExtraTrace (TraceUseBootstrapPeersChanged !_)) = GovernorEvent a
f a@(TraceOutboundGovernorCriticalFailure !_) = GovernorEvent a
f a@(TraceDebugState !_ !_) = GovernorEvent a
f a@(TraceChurnAction !_ !_ !_) = GovernorEvent a
f a@(TraceChurnTimeout !_ !_ !_) = GovernorEvent a
f a@(TraceVerifyPeerSnapshot !_) = GovernorEvent a
f a@(TraceLocalRootPeersChanged !_ !_) = GovernorEvent a
f a@(TraceTargetsChanged !_ !_) = GovernorEvent a
f a@(TracePublicRootsRequest !_ !_) = GovernorEvent a
f a@(TracePublicRootsResults !_ !_ !_) = GovernorEvent a
f a@(TracePublicRootsFailure !_ !_ !_) = GovernorEvent a
f a@(TraceForgetColdPeers !_ !_ !_) = GovernorEvent a
f a@(TraceBigLedgerPeersRequest !_ !_) = GovernorEvent a
f a@(TraceBigLedgerPeersResults !_ !_ !_) = GovernorEvent a
f a@(TraceBigLedgerPeersFailure !_ !_ !_) = GovernorEvent a
f a@(TraceForgetBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TracePickInboundPeers !_ !_ !_ !_) = GovernorEvent a
f a@(TracePeerShareRequests !_ !_ !_ !_ !_) = GovernorEvent a
f a@(TracePeerShareResults !_) = GovernorEvent a
f a@(TracePeerShareResultsFiltered !_) = GovernorEvent a
f a@(TracePromoteColdPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdLocalPeers !_ !_) = GovernorEvent a
f a@(TracePromoteColdFailed !_ !_ !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdBigLedgerPeerFailed !_ !_ !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteColdBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmLocalPeers !_ !_) = GovernorEvent a
f a@(TracePromoteWarmFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmAborted !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeerFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TracePromoteWarmBigLedgerPeerAborted !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmBigLedgerPeerFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteWarmBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteLocalHotPeers !_ !_) = GovernorEvent a
f a@(TraceDemoteHotFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotBigLedgerPeers !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotBigLedgerPeerFailed !_ !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteHotBigLedgerPeerDone !_ !_ !_) = GovernorEvent a
f a@(TraceDemoteAsynchronous !_) = GovernorEvent a
f a@(TraceDemoteLocalAsynchronous !_) = GovernorEvent a
f a@(TraceDemoteBigLedgerPeersAsynchronous !_) = GovernorEvent a
f a@TraceGovernorWakeup = GovernorEvent a
f a@(TraceChurnWait !_) = GovernorEvent a
f a@(ExtraTrace (TraceLedgerStateJudgementChanged !_)) = GovernorEvent a
f a@TraceOnlyBootstrapPeers = GovernorEvent a
f a@TraceBootstrapPeersFlagChangedWhilstInSensitiveState = GovernorEvent a
f a@(ExtraTrace (TraceUseBootstrapPeersChanged !_)) = GovernorEvent a
f a@(TraceOutboundGovernorCriticalFailure !_) = GovernorEvent a
f a@(TraceDebugState !_ !_) = GovernorEvent a
f a@(TraceChurnAction !_ !_ !_) = GovernorEvent a
f a@(TraceChurnTimeout !_ !_ !_) = GovernorEvent a
f a@(TraceVerifyPeerSnapshot !_) = GovernorEvent a

tracerDebugPeerSelection :: Tracer (IOSim s) (DebugPeerSelection Cardano.ExtraState PeerTrustable (Cardano.ExtraPeers PeerAddr) PeerAddr)
tracerDebugPeerSelection = GovernorDebug `contramap` tracerTestTraceEvent
Expand Down
4 changes: 4 additions & 0 deletions dmq-node/changelog.d/20251128_104246_karl.fb.knutsson_max_recon_main.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Breaking

- Enforce a maximum failure limit on cold promotions

1 change: 1 addition & 0 deletions dmq-node/src/DMQ/Diffusion/PeerSelection.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ policy gen =
, policyPeerShareBatchWaitTime = 0 -- seconds
, policyPeerShareOverallTimeout = 0 -- seconds
, policyPeerShareActivationDelay = 2 -- seconds
, policyMaxConnectionRetries = 5
}
where
pickTrivially :: Applicative m => Set SockAddr -> Int -> m (Set SockAddr)
Expand Down
4 changes: 4 additions & 0 deletions ouroboros-network/changelog.d/20251128_094205_karl.fb.knutsson_max_recon_main.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Breaking

- Enforce a maximum failure limit on cold promotions

3 changes: 2 additions & 1 deletion ouroboros-network/lib/Ouroboros/Network/Diffusion/Policies.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,8 @@ simplePeerSelectionPolicy rngVar metrics = PeerSelectionPolicy {
policyPeerShareRetryTime = 900, -- seconds
policyPeerShareBatchWaitTime = 3, -- seconds
policyPeerShareOverallTimeout = 10, -- seconds
policyPeerShareActivationDelay = 300 -- seconds
policyPeerShareActivationDelay = 300, -- seconds
policyMaxConnectionRetries = 5
}
where

Expand Down
36 changes: 26 additions & 10 deletions ouroboros-network/lib/Ouroboros/Network/PeerSelection/Governor/EstablishedPeers.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,11 +474,16 @@ jobPromoteColdPeer PeerSelectionActions {
peerStateActions = PeerStateActions {establishPeerConnection},
peerConnToPeerSharing,
extraPeersAPI = PublicExtraPeersAPI {
extraPeersToSet
extraPeersToSet,
memberExtraPeers,
differenceExtraPeers
},
extraStateToExtraCounters
}
PeerSelectionPolicy { policyPeerShareActivationDelay }
PeerSelectionPolicy {
policyPeerShareActivationDelay,
policyMaxConnectionRetries
}
peeraddr isBigLedgerPeer diffusionMode =
Job job handler () "promoteColdPeer"
where
Expand All @@ -487,6 +492,7 @@ jobPromoteColdPeer PeerSelectionActions {
extraTrace peeraddr peerconn)
handler e = return $
Completion $ \st@PeerSelectionState {
localRootPeers,
publicRootPeers,
stdGen,
targets = PeerSelectionTargets {
Expand All @@ -509,12 +515,22 @@ jobPromoteColdPeer PeerSelectionActions {
* 2 ^ (pred failCount `min` maxColdPeerRetryBackoff)
)
bigLedgerPeersSet = PublicRootPeers.getBigLedgerPeers publicRootPeers

st' = st { knownPeers = KnownPeers.setConnectTimes
(Map.singleton
peeraddr
(delay `addTime` now))
knownPeers',
unForgetAble = LocalRootPeers.member peeraddr localRootPeers ||
(memberExtraPeers peeraddr (PublicRootPeers.getExtraPeers publicRootPeers))
(publicRootPeers', knownPeers'', forgotten) =
if unForgetAble || failCount < policyMaxConnectionRetries
then ( publicRootPeers
, KnownPeers.setConnectTimes (Map.singleton peeraddr (delay `addTime` now))
knownPeers'
, False
)
else ( PublicRootPeers.difference differenceExtraPeers publicRootPeers
(Set.singleton peeraddr)
, KnownPeers.delete (Set.singleton peeraddr) knownPeers'
, True
)
st' = st { knownPeers = knownPeers'',
publicRootPeers = publicRootPeers',
inProgressPromoteCold = Set.delete peeraddr
(inProgressPromoteCold st),
stdGen = stdGen'
Expand All @@ -527,12 +543,12 @@ jobPromoteColdPeer PeerSelectionActions {
targetNumberOfEstablishedBigLedgerPeers
(case cs' of
PeerSelectionCounters { numberOfEstablishedBigLedgerPeers = a } -> a)
peeraddr delay e]
peeraddr delay e forgotten]
else [TracePromoteColdFailed
targetNumberOfEstablishedPeers
(case cs' of
PeerSelectionCounters { numberOfEstablishedPeers = a } -> a)
peeraddr delay e],
peeraddr delay e forgotten],
decisionState = st',
decisionJobs = []
}
Expand Down
Loading