Skip to content

OCPEDGE-1485: Add initial version of etcd kill and recovery test #30320

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

OCPEDGE-1485: Add initial version of etcd kill and recovery test #30320

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
15756df
Add initial version of etcd kill and recovery test
fonta-rh Sep 30, 2025
cff85ef
Simplify check
fonta-rh Oct 23, 2025
cf634ab
Refactor to better use Ginkgo recovery
fonta-rh Oct 24, 2025
993cf45
Add skip tag
fonta-rh Oct 24, 2025
af600b2
Update timeout values
fonta-rh Nov 13, 2025
f722ce0
Remove redundant eventually
fonta-rh Nov 13, 2025
939c27f
Remove obsolete comment and unnecessary true
fonta-rh Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion test/extended/two_node/tnf_recovery.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ type hypervisorExtendedConfig struct {
HypervisorKnownHostsPath string
}

var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Disruptive] Two Node with Fencing etcd recovery", func() {
var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Disruptive][Serial] Two Node with Fencing etcd recovery", func() {
defer g.GinkgoRecover()

var (
Expand Down Expand Up @@ -234,6 +234,29 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
&nodeB, true, false, // member on node B expected started == true, learner == false
membersHealthyAfterDoubleReboot, pollInterval)
})

g.It("should recover from etcd process crash [Skipped:KnownIssue]", func() {
// Note: This test kills the etcd process/container on one node to simulate
// a process crash, testing Pacemaker's ability to detect and restart etcd
// Currently skipped due to OCPBUGS-59238: rapid podman-etcd restart fails on unpatched clusters
g.GinkgoT().Printf("Randomly selected %s (%s) for etcd process crash and %s (%s) to survive\n",
targetNode.Name, targetNode.Status.Addresses[0].Address, peerNode.Name, peerNode.Status.Addresses[0].Address)

g.By(fmt.Sprintf("Killing etcd process/container on %s", targetNode.Name))
_, err := util.DebugNodeRetryWithOptionsAndChroot(oc, targetNode.Name, "openshift-etcd",
"bash", "-c", "podman kill etcd 2>/dev/null")
o.Expect(err).To(o.BeNil(), "Expected to kill etcd process without command errors")

g.By("Waiting for cluster to recover - both nodes become started voting members")
// Retry validation with 45-second intervals, up to 8 attempts (6 minutes total)
defer g.GinkgoRecover()

validateEtcdRecoveryState(oc, etcdClientFactory,
&peerNode,
&targetNode, true, false, // targetNode expected started == true, learner == false
6*time.Minute, 45*time.Second)

})
})

func getMembers(etcdClientFactory helpers.EtcdClientCreator) ([]*etcdserverpb.Member, error) {
Expand Down