33package tests
44
55import (
6+ "encoding/json"
67 "fmt"
8+ "net"
9+ "os/exec"
710 "time"
811
912 "github.com/onsi/ginkgo/v2"
@@ -18,6 +21,7 @@ import (
1821 "github.com/metallb/frrk8stests/pkg/k8sclient"
1922 "github.com/metallb/frrk8stests/pkg/routes"
2023 . "github.com/onsi/gomega"
24+ "go.universe.tf/e2etest/pkg/executor"
2125 frrconfig "go.universe.tf/e2etest/pkg/frr/config"
2226 "go.universe.tf/e2etest/pkg/ipfamily"
2327 corev1 "k8s.io/api/core/v1"
@@ -68,19 +72,29 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
6872 dump .K8sInfo (testName , reporter )
6973 dump .BGPInfo (testName , infra .FRRContainers , cs )
7074 }
75+
76+ err := cleanup (updater )
77+ Expect (err ).NotTo (HaveOccurred (), "cleanup config in API and infra containers" )
7178 })
7279
7380 ginkgo .Context ("When restarting the frrk8s deamon pods" , func () {
7481
75- ginkgo .DescribeTable ("external BGP peer maintains routes" , func (ipFam ipfamily.Family , prefix string ) {
82+ ginkgo .DescribeTable ("external BGP peer maintains routes" , func (ipFam ipfamily.Family , prefix , learnRoute string ) {
83+ cc := func (frr * container.FRR ) {
84+ if ipFam == ipfamily .IPv4 {
85+ frr .NeighborConfig .ToAdvertiseV4 = []string {learnRoute }
86+ } else {
87+ frr .NeighborConfig .ToAdvertiseV6 = []string {learnRoute }
88+ }
89+ }
7690 frrs := config .ContainersForVRF (infra .FRRContainers , "" )
7791 for _ , c := range frrs {
78- err := container .PairWithNodes (cs , c , ipFam )
92+ err := container .PairWithNodes (cs , c , ipFam , cc )
7993 Expect (err ).NotTo (HaveOccurred (), "set frr config in infra containers failed" )
8094 }
8195
8296 peersConfig := config .PeersForContainers (frrs , ipFam ,
83- config .EnableAllowAll , config .EnableGracefulRestart )
97+ config .EnableAllowAll , config .EnableReceiveAllowAll , config . EnableGracefulRestart )
8498
8599 frrConfigCR := frrk8sv1beta1.FRRConfiguration {
86100 ObjectMeta : metav1.ObjectMeta {
@@ -110,12 +124,16 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
110124 return fmt .Errorf ("Neigh %s does not have prefix %s: %w" , p .FRR .Name , prefix , err )
111125 }
112126 }
127+ if err := checkRoutesOnNodes (nodes , learnRoute ); err != nil {
128+ return err
129+ }
113130 return nil
114131 }
115132
116133 Eventually (check , time .Minute , time .Second ).ShouldNot (HaveOccurred (),
117134 "route should exist before we restart frr-k8s" )
118135
136+ ginkgo .By ("GR started" )
119137 c := make (chan struct {})
120138 go func () { // go restart frr-k8s while Consistently check that route exists
121139 defer ginkgo .GinkgoRecover ()
@@ -128,8 +146,53 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
128146 Consistently (check , 2 * time .Minute , time .Second ).ShouldNot (HaveOccurred ())
129147 Eventually (c , time .Minute , time .Second ).Should (BeClosed (), "restart FRRK8s pods are not yet ready" )
130148 },
131- ginkgo .Entry ("IPV4" , ipfamily .IPv4 , "192.168.2.0/24" ),
132- ginkgo .Entry ("IPV6" , ipfamily .IPv6 , "fc00:f853:ccd:e799::/64" ),
149+ ginkgo .Entry ("IPV4" , ipfamily .IPv4 , "192.168.2.0/24" , "200.200.200.0/24" ),
150+ ginkgo .Entry ("IPV6" , ipfamily .IPv6 , "fc00:f853:ccd:e799::/64" , "2001:db8::/64" ),
133151 )
134152 })
135153})
154+
155+ type IPRouteNexthop struct {
156+ Gateway string `json:"gateway"`
157+ Dev string `json:"dev"`
158+ Weight int `json:"weight"`
159+ Flags []any `json:"flags"`
160+ }
161+
162+ type IPRoute struct {
163+ Dst string `json:"dst"`
164+ Nhid int `json:"nhid"`
165+ Metric int `json:"metric"`
166+ Flags []any `json:"flags"`
167+ Nexthops []IPRouteNexthop `json:"nexthops"`
168+ }
169+
170+ func checkRoutesOnNodes (nodes []corev1.Node , want string ) error {
171+ isIPv6 := func (address string ) bool {
172+ ip , _ , _ := net .ParseCIDR (address )
173+ return ip != nil && ip .To4 () == nil // Valid IP and not an IPv4 address
174+ }
175+
176+ for _ , n := range nodes {
177+ cmd := "ip --json route show proto bgp"
178+ if isIPv6 (want ) {
179+ cmd = "ip --json -6 route show proto bgp"
180+ }
181+ out , err := exec .Command (executor .ContainerRuntime ,
182+ "exec" , n .GetName (), "/bin/bash" , "-c" , cmd ).CombinedOutput ()
183+ if err != nil {
184+ return fmt .Errorf ("%s - %w" , out , err )
185+ }
186+ var routes []IPRoute
187+ if err := json .Unmarshal ([]byte (out ), & routes ); err != nil {
188+ return fmt .Errorf ("Error unmarshaling JSON: %v" , err )
189+ }
190+ for _ , r := range routes {
191+ if r .Dst == want {
192+ return nil
193+ }
194+ }
195+ return fmt .Errorf ("local k8s node route %s was not found" , want )
196+ }
197+ return nil
198+ }
0 commit comments