33package tests
44
55import (
6+ "errors"
67 "fmt"
78 "time"
89
910 "github.com/onsi/ginkgo/v2"
1011 "github.com/openshift-kni/k8sreporter"
1112 "go.universe.tf/e2etest/pkg/frr/container"
13+ frrcontainer "go.universe.tf/e2etest/pkg/frr/container"
14+
15+ "go.universe.tf/e2etest/pkg/executor"
16+ "go.universe.tf/e2etest/pkg/frr"
1217
1318 frrk8sv1beta1 "github.com/metallb/frr-k8s/api/v1beta1"
1419 "github.com/metallb/frrk8stests/pkg/config"
@@ -28,10 +33,11 @@ import (
2833
2934var _ = ginkgo .Describe ("Establish BGP session with EnableGracefulRestart" , func () {
3035 var (
31- cs clientset.Interface
32- updater * config.Updater
33- reporter * k8sreporter.KubernetesReporter
34- nodes []corev1.Node
36+ cs clientset.Interface
37+ updater * config.Updater
38+ reporter * k8sreporter.KubernetesReporter
39+ nodes []corev1.Node
40+ prefixesV4 = scaleUP (200 )
3541 )
3642
3743 cleanup := func (u * config.Updater ) error {
@@ -63,17 +69,23 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
6369 })
6470
6571 ginkgo .AfterEach (func () {
72+
73+ seed := ginkgo .GinkgoRandomSeed ()
74+ testName := fmt .Sprintf ("%s-%d" , ginkgo .CurrentSpecReport ().LeafNodeText , seed )
6675 if ginkgo .CurrentSpecReport ().Failed () {
67- testName := ginkgo .CurrentSpecReport ().LeafNodeText
68- dump .K8sInfo (testName , reporter )
69- dump .BGPInfo (testName , infra .FRRContainers , cs )
76+ testName += "-failed"
7077 }
78+ ginkgo .By (testName )
79+ dump .K8sInfo (testName , reporter )
80+ dump .BGPInfo (testName , infra .FRRContainers , cs )
7181 })
7282
7383 ginkgo .Context ("When restarting the frrk8s deamon pods" , func () {
7484
75- ginkgo .DescribeTable ("external BGP peer maintains routes" , func (ipFam ipfamily.Family , prefix string ) {
85+ ginkgo .DescribeTable ("external BGP peer maintains routes" , func (ipFam ipfamily.Family , prefix [] string ) {
7686 frrs := config .ContainersForVRF (infra .FRRContainers , "" )
87+ // cnt, err := config.ContainerByName(infra.FRRContainers, "ebgp-multi-hop")
88+ // frrs := []*frrcontainer.FRR{cnt}
7789 for _ , c := range frrs {
7890 err := container .PairWithNodes (cs , c , ipFam )
7991 Expect (err ).NotTo (HaveOccurred (), "set frr config in infra containers failed" )
@@ -88,48 +100,145 @@ var _ = ginkgo.Describe("Establish BGP session with EnableGracefulRestart", func
88100 Namespace : k8s .FRRK8sNamespace ,
89101 },
90102 Spec : frrk8sv1beta1.FRRConfigurationSpec {
103+ // NodeSelector: metav1.LabelSelector{
104+ // MatchLabels: map[string]string{
105+ // "kubernetes.io/hostname": nodes[0].GetLabels()["kubernetes.io/hostname"],
106+ // },
107+ // },
91108 BGP : frrk8sv1beta1.BGPConfig {
92109 Routers : []frrk8sv1beta1.Router {
93110 {
94111 ASN : infra .FRRK8sASN ,
95112 Neighbors : config .NeighborsFromPeers (peersConfig .PeersV4 , peersConfig .PeersV6 ),
96- Prefixes : [] string { prefix } ,
113+ Prefixes : prefix ,
97114 },
98115 },
99116 },
100117 },
101118 }
119+ ginkgo .By ("Before GR test" )
102120
103121 err := updater .Update (peersConfig .Secrets , frrConfigCR )
104122 Expect (err ).NotTo (HaveOccurred (), "apply the CR in k8s api failed" )
105-
106- check := func () error {
123+ Eventually (func () error {
107124 for _ , p := range peersConfig .Peers () {
108- err := routes .CheckNeighborHasPrefix (p .FRR , p .FRR .RouterConfig .VRF , prefix , nodes )
125+ ValidateFRRPeeredWithNodes (nodes , & p .FRR , ipFam )
126+ neighbors , err := frr .NeighborsInfo (p .FRR )
127+ Expect (err ).NotTo (HaveOccurred ())
128+ for _ , n := range neighbors {
129+ Expect (n .GRInfo .RemoteGrMode ).Should (Equal ("Restart" ))
130+ }
131+
132+ err = routes .CheckNeighborHasPrefix (p .FRR , p .FRR .RouterConfig .VRF , prefix [0 ], nodes )
109133 if err != nil {
110- return fmt .Errorf ("Neigh %s does not have prefix %s : %w" , p .FRR .Name , prefix , err )
134+ return fmt .Errorf ("Neigh %s does not have prefixes : %w" , p .FRR .Name , err )
111135 }
112136 }
113137 return nil
114- }
115-
116- Eventually (check , time .Minute , time .Second ).ShouldNot (HaveOccurred (),
117- "route should exist before we restart frr-k8s" )
138+ }, time .Minute , time .Second ).ShouldNot (HaveOccurred (), "route should exist before we restart frr-k8s" )
118139
140+ ginkgo .By ("Start GR test" )
119141 c := make (chan struct {})
120142 go func () { // go restart frr-k8s while Consistently check that route exists
121143 defer ginkgo .GinkgoRecover ()
122144 err := k8s .RestartFRRK8sPods (cs )
123145 Expect (err ).NotTo (HaveOccurred (), "frr-k8s pods failed to restart" )
146+ for _ , p := range peersConfig .Peers () {
147+ ValidateFRRPeeredWithNodes (nodes , & p .FRR , ipFam )
148+ }
149+ ginkgo .By ("FRRK8s pod restarted and BGP established" )
124150 close (c )
125151 }()
126152
153+ check := func () error {
154+ var returnError error
155+
156+ for _ , p := range peersConfig .Peers () {
157+ err := checkRoutes (p .FRR , prefix )
158+ if err != nil {
159+ returnError = errors .Join (returnError , fmt .Errorf ("Neigh %s : %w" , p .FRR .Name , err ))
160+ for i := 0 ; i < 20 ; i ++ {
161+ if err := checkRoutes (p .FRR , prefix ); err != nil {
162+ ginkgo .By (fmt .Sprintf ("%d Neigh %s does NOT have prefix %v" , i , p .FRR .Name , err ))
163+ } else {
164+ ginkgo .By (fmt .Sprintf ("%d Neigh %s does have prefix" , i , p .FRR .Name ))
165+ }
166+ time .Sleep (time .Second )
167+ }
168+ }
169+ }
170+ return returnError
171+ }
172+
127173 // 2*time.Minute is important because that is the Graceful Restart timer.
128- Consistently (check , 2 * time .Minute , time .Second ).ShouldNot (HaveOccurred ())
174+ Consistently (check , 30 * time .Second , time .Second ).ShouldNot (HaveOccurred ())
129175 Eventually (c , time .Minute , time .Second ).Should (BeClosed (), "restart FRRK8s pods are not yet ready" )
130176 },
131- ginkgo .Entry ("IPV4" , ipfamily .IPv4 , "192.168.2.0/24" ),
132- ginkgo .Entry ("IPV6" , ipfamily .IPv6 , "fc00:f853:ccd:e799::/64" ),
177+ ginkgo .Entry ("IPV4" , ipfamily .IPv4 , prefixesV4 ),
178+ // ginkgo.Entry("IPV6", ipfamily.IPv6, []string{"2001:db8:5555::5/128"} ),
133179 )
134180 })
135181})
182+
183+ func checkRoutes (cnt frrcontainer.FRR , want []string ) error {
184+ m := sliceToMap (want )
185+ v4 , _ , err := frr .Routes (cnt )
186+ if err != nil {
187+ // ignore the docker exec errors
188+ return nil
189+ }
190+ if len (m ) == 0 {
191+ return fmt .Errorf ("nil map m" )
192+ }
193+ if len (v4 ) == 0 {
194+ IPRoutes (cnt )
195+ return fmt .Errorf ("nil map v4" )
196+ }
197+ for _ , r := range v4 {
198+ // if r.Stale {
199+ // fmt.Printf("S")
200+ // }
201+ delete (m , r .Destination .String ())
202+ }
203+ if len (m ) > 0 {
204+ return fmt .Errorf ("%d routes %+v not found " , len (m ), getKeys (m ))
205+ }
206+ return nil
207+ }
208+
209+ func scaleUP (size int ) []string {
210+ if size > 255 {
211+ panic ("255 is max" )
212+ }
213+
214+ ret := []string {}
215+ for i := 0 ; i < size ; i ++ {
216+ ret = append (ret , fmt .Sprintf ("5.5.5.%d/32" , i ))
217+ }
218+
219+ return ret
220+ }
221+
222+ func sliceToMap (slice []string ) map [string ]bool {
223+ m := make (map [string ]bool )
224+ for _ , v := range slice {
225+ m [v ] = true
226+ }
227+ return m
228+ }
229+ func getKeys (m map [string ]bool ) []string {
230+ keys := make ([]string , 0 , len (m )) // Initialize slice with the capacity of the map length
231+ for key := range m {
232+ keys = append (keys , key )
233+ }
234+ return keys
235+ }
236+ func IPRoutes (exec executor.Executor ) error {
237+ cmd := "show ip route bgp"
238+ res , err := exec .Exec ("vtysh" , "-c" , cmd )
239+ if err != nil {
240+ return errors .Join (err , errors .New ("Failed to query routes" ))
241+ }
242+ fmt .Println ("res" , res )
243+ return nil
244+ }
0 commit comments