Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
wip: run many times test to flake
Browse files Browse the repository at this point in the history
Signed-off-by: karampok <karampok@gmail.com>
karampok committed Oct 29, 2024
1 parent bfeacb8 commit e78d8e2
Showing 24 changed files with 51 additions and 11 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -238,7 +238,7 @@ $(APIDOCSGEN): $(LOCALBIN)

.PHONY: e2etests
e2etests: ginkgo kubectl
$(GINKGO) -v $(GINKGO_ARGS) --timeout=3h ./e2etests -- --kubectl=$(KUBECTL) $(TEST_ARGS)
$(GINKGO) -v $(GINKGO_ARGS) --repeat=50 --timeout=3h ./e2etests -- --kubectl=$(KUBECTL) $(TEST_ARGS)


.PHONY: kind-cluster
1 change: 1 addition & 0 deletions e2etests/pkg/k8s/pods.go
Original file line number Diff line number Diff line change
@@ -74,6 +74,7 @@ func RestartFRRK8sPods(cs clientset.Interface) error {
return false, nil
}
}
ginkgo.By(fmt.Sprintf("\tfrrk8s pod ARE ready"))
return true, nil
})
}
35 changes: 27 additions & 8 deletions e2etests/tests/graceful_restart.go
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
package tests

import (
"errors"
"fmt"
"time"

@@ -26,7 +27,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
)

var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func() {
var _ = ginkgo.FDescribe("Establish BGP session with EnableGracefulRestart", func() {
var (
cs clientset.Interface
updater *config.Updater
@@ -103,19 +104,17 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
err := updater.Update(peersConfig.Secrets, frrConfigCR)
Expect(err).NotTo(HaveOccurred(), "apply the CR in k8s api failed")

check := func() error {
Eventually(func() error {
for _, p := range peersConfig.Peers() {
err := routes.CheckNeighborHasPrefix(p.FRR, p.FRR.RouterConfig.VRF, prefix, nodes)
if err != nil {
return fmt.Errorf("Neigh %s does not have prefix %s: %w", p.FRR.Name, prefix, err)
}
}
return nil
}

Eventually(check, time.Minute, time.Second).ShouldNot(HaveOccurred(),
"route should exist before we restart frr-k8s")
}, time.Minute, time.Second).ShouldNot(HaveOccurred(), "route should exist before we restart frr-k8s")

time.Sleep(5 * time.Second)
c := make(chan struct{})
go func() { // go restart frr-k8s while Consistently check that route exists
defer ginkgo.GinkgoRecover()
@@ -124,12 +123,32 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
close(c)
}()

check := func() error {
var returnError error

for _, p := range peersConfig.Peers() {
err = routes.CheckNeighborHasPrefix(p.FRR, p.FRR.RouterConfig.VRF, prefix, nodes)
if err != nil {
returnError = errors.Join(returnError, fmt.Errorf("Neigh %s does not have prefix %s: %w", p.FRR.Name, prefix, err))
for i := 0; i < 5; i++ {
if err := routes.CheckNeighborHasPrefix(p.FRR, p.FRR.RouterConfig.VRF, prefix, nodes); err != nil {
ginkgo.By(fmt.Sprintf("%d Neigh %s does not have prefix %s: %s", i, p.FRR.Name, prefix, err))
time.Sleep(time.Second)
} else {
ginkgo.By(fmt.Sprintf("%d Neigh %s does have prefix %s: %s", i, p.FRR.Name, prefix, err))
}
}
}
}
return returnError
}

// 2*time.Minute is important because that is the Graceful Restart timer.
Consistently(check, 2*time.Minute, time.Second).ShouldNot(HaveOccurred())
Eventually(c, time.Minute, time.Second).Should(BeClosed(), "restart FRRK8s pods are not yet ready")
},
ginkgo.Entry("IPV4", ipfamily.IPv4, "192.168.2.0/24"),
ginkgo.Entry("IPV6", ipfamily.IPv6, "fc00:f853:ccd:e799::/64"),
ginkgo.Entry("IPV4", ipfamily.IPv4, "5.5.5.5/32"),
ginkgo.Entry("IPV6", ipfamily.IPv6, "2001:db8:5555::5/128"),
)
})
})
4 changes: 2 additions & 2 deletions frr-tools/reloader/frr-reloader.sh
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ reload_frr() {
kill_sleep

echo "Checking the configuration file syntax"
if ! python3 /usr/lib/frr/frr-reload.py --test --stdout "$FILE_TO_RELOAD" 2>"$LAST_ERROR_FILE" | sed 's/password.*/password <retracted>/g'; then
if ! python3 /usr/lib/frr/frr-reload.py --log-level=debug --test --stdout "$FILE_TO_RELOAD" 2>"$LAST_ERROR_FILE" | sed 's/password.*/password <retracted>/g'; then
echo "Syntax error spotted: aborting.. $SECONDS seconds"
cat "$LAST_ERROR_FILE" | sed 's/password.*/password <retracted>/g'
echo -n "$(date +%s) failure" > "$STATUSFILE"
@@ -26,7 +26,7 @@ reload_frr() {
fi

echo "Applying the configuration file"
if ! python3 /usr/lib/frr/frr-reload.py --reload --overwrite --stdout "$FILE_TO_RELOAD" 2>&1 | sed 's/password.*/password <retracted>/g'; then
if ! python3 /usr/lib/frr/frr-reload.py --log-level=debug --reload --overwrite --stdout "$FILE_TO_RELOAD" 2>&1 | sed 's/password.*/password <retracted>/g'; then
echo "Failed to fully apply configuration file $SECONDS seconds"
echo -n "$(date +%s) failure" > "$STATUSFILE"
save_status
1 change: 1 addition & 0 deletions internal/frr/templates/frr.tmpl
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ debug route-map detail
hostname {{.Hostname}}
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0

{{- range $r := .Routers }}
{{- range .Neighbors }}
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



1 change: 1 addition & 0 deletions internal/frr/testdata/TestMultipleRoutersImportVRFs.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



1 change: 1 addition & 0 deletions internal/frr/testdata/TestSingleSession
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



1 change: 1 addition & 0 deletions internal/frr/testdata/TestSingleSession.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



1 change: 1 addition & 0 deletions internal/frr/testdata/TestSingleSessionBFD.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



1 change: 1 addition & 0 deletions internal/frr/testdata/TestTwoRoutersTwoNeighbors.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0



1 change: 1 addition & 0 deletions internal/frr/testdata/TestTwoRoutersTwoNeighborsBFD.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
1 change: 1 addition & 0 deletions internal/frr/testdata/TestTwoSessionsAcceptAll.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
1 change: 1 addition & 0 deletions internal/frr/testdata/TestTwoSessionsAcceptSomeV4.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1
1 change: 1 addition & 0 deletions internal/frr/testdata/TestTwoSessionsAcceptV4AndV6.golden
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ log timestamp precision 3
hostname dummyhostname
ip nht resolve-via-default
ipv6 nht resolve-via-default
bgp route-map delay-timer 0


route-map 192.168.1.2-out permit 1

0 comments on commit e78d8e2

Please sign in to comment.