From e78d8e2674f6709ed3f9189f2e0e4a951d7111d1 Mon Sep 17 00:00:00 2001 From: karampok Date: Tue, 29 Oct 2024 13:12:40 +0100 Subject: [PATCH] wip: run many times test to flake Signed-off-by: karampok --- Makefile | 2 +- e2etests/pkg/k8s/pods.go | 1 + e2etests/tests/graceful_restart.go | 35 ++++++++++++++----- frr-tools/reloader/frr-reloader.sh | 4 +-- internal/frr/templates/frr.tmpl | 1 + .../TestMultipleNeighborsOneV4AndOneV6.golden | 1 + ...ipleNeighborsOneV4AndOneV6DisableMP.golden | 1 + .../TestMultipleRoutersImportVRFs.golden | 1 + .../TestMultipleRoutersMultipleNeighs.golden | 1 + internal/frr/testdata/TestSingleSession | 1 + .../frr/testdata/TestSingleSession.golden | 1 + .../frr/testdata/TestSingleSessionBFD.golden | 1 + .../TestSingleSessionWithAlwaysBlock.golden | 1 + .../TestSingleSessionWithEBGPMultihop.golden | 1 + ...gleSessionWithEBGPMultihopAndExtras.golden | 1 + .../TestSingleSessionWithExternalASN.golden | 1 + ...estSingleSessionWithGracefulRestart.golden | 1 + .../TestSingleSessionWithIPv6SingleHop.golden | 1 + .../TestSingleSessionWithInternalASN.golden | 1 + .../TestTwoRoutersTwoNeighbors.golden | 1 + .../TestTwoRoutersTwoNeighborsBFD.golden | 1 + .../testdata/TestTwoSessionsAcceptAll.golden | 1 + .../TestTwoSessionsAcceptSomeV4.golden | 1 + .../TestTwoSessionsAcceptV4AndV6.golden | 1 + 24 files changed, 51 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 31b445b1..2dd42915 100644 --- a/Makefile +++ b/Makefile @@ -238,7 +238,7 @@ $(APIDOCSGEN): $(LOCALBIN) .PHONY: e2etests e2etests: ginkgo kubectl - $(GINKGO) -v $(GINKGO_ARGS) --timeout=3h ./e2etests -- --kubectl=$(KUBECTL) $(TEST_ARGS) + $(GINKGO) -v $(GINKGO_ARGS) --repeat=50 --timeout=3h ./e2etests -- --kubectl=$(KUBECTL) $(TEST_ARGS) .PHONY: kind-cluster diff --git a/e2etests/pkg/k8s/pods.go b/e2etests/pkg/k8s/pods.go index cbddfdea..860b0048 100644 --- a/e2etests/pkg/k8s/pods.go +++ b/e2etests/pkg/k8s/pods.go @@ -74,6 +74,7 @@ func RestartFRRK8sPods(cs clientset.Interface) error { return false, nil } } + ginkgo.By(fmt.Sprintf("\tfrrk8s pod ARE ready")) return true, nil }) } diff --git a/e2etests/tests/graceful_restart.go b/e2etests/tests/graceful_restart.go index cfea006e..2b5f342c 100644 --- a/e2etests/tests/graceful_restart.go +++ b/e2etests/tests/graceful_restart.go @@ -3,6 +3,7 @@ package tests import ( + "errors" "fmt" "time" @@ -26,7 +27,7 @@ import ( clientset "k8s.io/client-go/kubernetes" ) -var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func() { +var _ = ginkgo.FDescribe("Establish BGP session with EnableGracefulRestart", func() { var ( cs clientset.Interface updater *config.Updater @@ -103,7 +104,7 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func err := updater.Update(peersConfig.Secrets, frrConfigCR) Expect(err).NotTo(HaveOccurred(), "apply the CR in k8s api failed") - check := func() error { + Eventually(func() error { for _, p := range peersConfig.Peers() { err := routes.CheckNeighborHasPrefix(p.FRR, p.FRR.RouterConfig.VRF, prefix, nodes) if err != nil { @@ -111,11 +112,9 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func } } return nil - } - - Eventually(check, time.Minute, time.Second).ShouldNot(HaveOccurred(), - "route should exist before we restart frr-k8s") + }, time.Minute, time.Second).ShouldNot(HaveOccurred(), "route should exist before we restart frr-k8s") + time.Sleep(5 * time.Second) c := make(chan struct{}) go func() { // go restart frr-k8s while Consistently check that route exists defer ginkgo.GinkgoRecover() @@ -124,12 +123,32 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func close(c) }() + check := func() error { + var returnError error + + for _, p := range peersConfig.Peers() { + err = routes.CheckNeighborHasPrefix(p.FRR, p.FRR.RouterConfig.VRF, prefix, nodes) + if err != nil { + returnError = errors.Join(returnError, fmt.Errorf("Neigh %s does not have prefix %s: %w", p.FRR.Name, prefix, err)) + for i := 0; i < 5; i++ { + if err := routes.CheckNeighborHasPrefix(p.FRR, p.FRR.RouterConfig.VRF, prefix, nodes); err != nil { + ginkgo.By(fmt.Sprintf("%d Neigh %s does not have prefix %s: %s", i, p.FRR.Name, prefix, err)) + time.Sleep(time.Second) + } else { + ginkgo.By(fmt.Sprintf("%d Neigh %s does have prefix %s: %s", i, p.FRR.Name, prefix, err)) + } + } + } + } + return returnError + } + // 2*time.Minute is important because that is the Graceful Restart timer. Consistently(check, 2*time.Minute, time.Second).ShouldNot(HaveOccurred()) Eventually(c, time.Minute, time.Second).Should(BeClosed(), "restart FRRK8s pods are not yet ready") }, - ginkgo.Entry("IPV4", ipfamily.IPv4, "192.168.2.0/24"), - ginkgo.Entry("IPV6", ipfamily.IPv6, "fc00:f853:ccd:e799::/64"), + ginkgo.Entry("IPV4", ipfamily.IPv4, "5.5.5.5/32"), + ginkgo.Entry("IPV6", ipfamily.IPv6, "2001:db8:5555::5/128"), ) }) }) diff --git a/frr-tools/reloader/frr-reloader.sh b/frr-tools/reloader/frr-reloader.sh index 2bb56e1f..945ced2b 100755 --- a/frr-tools/reloader/frr-reloader.sh +++ b/frr-tools/reloader/frr-reloader.sh @@ -17,7 +17,7 @@ reload_frr() { kill_sleep echo "Checking the configuration file syntax" - if ! python3 /usr/lib/frr/frr-reload.py --test --stdout "$FILE_TO_RELOAD" 2>"$LAST_ERROR_FILE" | sed 's/password.*/password /g'; then + if ! python3 /usr/lib/frr/frr-reload.py --log-level=debug --test --stdout "$FILE_TO_RELOAD" 2>"$LAST_ERROR_FILE" | sed 's/password.*/password /g'; then echo "Syntax error spotted: aborting.. $SECONDS seconds" cat "$LAST_ERROR_FILE" | sed 's/password.*/password /g' echo -n "$(date +%s) failure" > "$STATUSFILE" @@ -26,7 +26,7 @@ reload_frr() { fi echo "Applying the configuration file" - if ! python3 /usr/lib/frr/frr-reload.py --reload --overwrite --stdout "$FILE_TO_RELOAD" 2>&1 | sed 's/password.*/password /g'; then + if ! python3 /usr/lib/frr/frr-reload.py --log-level=debug --reload --overwrite --stdout "$FILE_TO_RELOAD" 2>&1 | sed 's/password.*/password /g'; then echo "Failed to fully apply configuration file $SECONDS seconds" echo -n "$(date +%s) failure" > "$STATUSFILE" save_status diff --git a/internal/frr/templates/frr.tmpl b/internal/frr/templates/frr.tmpl index 00546c32..049b8354 100644 --- a/internal/frr/templates/frr.tmpl +++ b/internal/frr/templates/frr.tmpl @@ -19,6 +19,7 @@ debug route-map detail hostname {{.Hostname}} ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 {{- range $r := .Routers }} {{- range .Neighbors }} diff --git a/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6.golden b/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6.golden index 3a51b1ef..c5c43321 100644 --- a/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6.golden +++ b/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6DisableMP.golden b/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6DisableMP.golden index afa9a5b0..d72ba18f 100644 --- a/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6DisableMP.golden +++ b/internal/frr/testdata/TestMultipleNeighborsOneV4AndOneV6DisableMP.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestMultipleRoutersImportVRFs.golden b/internal/frr/testdata/TestMultipleRoutersImportVRFs.golden index 63089ad7..df760fc3 100644 --- a/internal/frr/testdata/TestMultipleRoutersImportVRFs.golden +++ b/internal/frr/testdata/TestMultipleRoutersImportVRFs.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestMultipleRoutersMultipleNeighs.golden b/internal/frr/testdata/TestMultipleRoutersMultipleNeighs.golden index 2eac7cc2..76de9e07 100644 --- a/internal/frr/testdata/TestMultipleRoutersMultipleNeighs.golden +++ b/internal/frr/testdata/TestMultipleRoutersMultipleNeighs.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSession b/internal/frr/testdata/TestSingleSession index 00900d80..7255b65f 100644 --- a/internal/frr/testdata/TestSingleSession +++ b/internal/frr/testdata/TestSingleSession @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSession.golden b/internal/frr/testdata/TestSingleSession.golden index 00900d80..7255b65f 100644 --- a/internal/frr/testdata/TestSingleSession.golden +++ b/internal/frr/testdata/TestSingleSession.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSessionBFD.golden b/internal/frr/testdata/TestSingleSessionBFD.golden index 7cb4e446..fcb4fd3d 100644 --- a/internal/frr/testdata/TestSingleSessionBFD.golden +++ b/internal/frr/testdata/TestSingleSessionBFD.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestSingleSessionWithAlwaysBlock.golden b/internal/frr/testdata/TestSingleSessionWithAlwaysBlock.golden index b9812d15..8f79054b 100644 --- a/internal/frr/testdata/TestSingleSessionWithAlwaysBlock.golden +++ b/internal/frr/testdata/TestSingleSessionWithAlwaysBlock.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestSingleSessionWithEBGPMultihop.golden b/internal/frr/testdata/TestSingleSessionWithEBGPMultihop.golden index 8a63064d..cdf80533 100644 --- a/internal/frr/testdata/TestSingleSessionWithEBGPMultihop.golden +++ b/internal/frr/testdata/TestSingleSessionWithEBGPMultihop.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSessionWithEBGPMultihopAndExtras.golden b/internal/frr/testdata/TestSingleSessionWithEBGPMultihopAndExtras.golden index c6005c0c..1acd65bb 100644 --- a/internal/frr/testdata/TestSingleSessionWithEBGPMultihopAndExtras.golden +++ b/internal/frr/testdata/TestSingleSessionWithEBGPMultihopAndExtras.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSessionWithExternalASN.golden b/internal/frr/testdata/TestSingleSessionWithExternalASN.golden index 94459372..6d10df91 100644 --- a/internal/frr/testdata/TestSingleSessionWithExternalASN.golden +++ b/internal/frr/testdata/TestSingleSessionWithExternalASN.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSessionWithGracefulRestart.golden b/internal/frr/testdata/TestSingleSessionWithGracefulRestart.golden index 9c957ad9..440c5cbe 100644 --- a/internal/frr/testdata/TestSingleSessionWithGracefulRestart.golden +++ b/internal/frr/testdata/TestSingleSessionWithGracefulRestart.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestSingleSessionWithIPv6SingleHop.golden b/internal/frr/testdata/TestSingleSessionWithIPv6SingleHop.golden index 775843a1..66a31463 100644 --- a/internal/frr/testdata/TestSingleSessionWithIPv6SingleHop.golden +++ b/internal/frr/testdata/TestSingleSessionWithIPv6SingleHop.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestSingleSessionWithInternalASN.golden b/internal/frr/testdata/TestSingleSessionWithInternalASN.golden index 8bc6ac21..8ea1b2eb 100644 --- a/internal/frr/testdata/TestSingleSessionWithInternalASN.golden +++ b/internal/frr/testdata/TestSingleSessionWithInternalASN.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestTwoRoutersTwoNeighbors.golden b/internal/frr/testdata/TestTwoRoutersTwoNeighbors.golden index 747db855..57ca837f 100644 --- a/internal/frr/testdata/TestTwoRoutersTwoNeighbors.golden +++ b/internal/frr/testdata/TestTwoRoutersTwoNeighbors.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 diff --git a/internal/frr/testdata/TestTwoRoutersTwoNeighborsBFD.golden b/internal/frr/testdata/TestTwoRoutersTwoNeighborsBFD.golden index 3deb4473..b2623cf5 100644 --- a/internal/frr/testdata/TestTwoRoutersTwoNeighborsBFD.golden +++ b/internal/frr/testdata/TestTwoRoutersTwoNeighborsBFD.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestTwoSessionsAcceptAll.golden b/internal/frr/testdata/TestTwoSessionsAcceptAll.golden index 2ec0fb48..a1ff047a 100644 --- a/internal/frr/testdata/TestTwoSessionsAcceptAll.golden +++ b/internal/frr/testdata/TestTwoSessionsAcceptAll.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestTwoSessionsAcceptSomeV4.golden b/internal/frr/testdata/TestTwoSessionsAcceptSomeV4.golden index b15edbd6..ec652d67 100644 --- a/internal/frr/testdata/TestTwoSessionsAcceptSomeV4.golden +++ b/internal/frr/testdata/TestTwoSessionsAcceptSomeV4.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1 diff --git a/internal/frr/testdata/TestTwoSessionsAcceptV4AndV6.golden b/internal/frr/testdata/TestTwoSessionsAcceptV4AndV6.golden index 811776b7..288c1480 100644 --- a/internal/frr/testdata/TestTwoSessionsAcceptV4AndV6.golden +++ b/internal/frr/testdata/TestTwoSessionsAcceptV4AndV6.golden @@ -3,6 +3,7 @@ log timestamp precision 3 hostname dummyhostname ip nht resolve-via-default ipv6 nht resolve-via-default +bgp route-map delay-timer 0 route-map 192.168.1.2-out permit 1