From 7221563037693a95899a3cb42386301bdaf4b550 Mon Sep 17 00:00:00 2001 From: Federico Paolinelli Date: Fri, 7 Jul 2023 12:01:39 +0300 Subject: [PATCH 1/3] Makefile: fix load on kind to use the local binary Signed-off-by: Federico Paolinelli --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 98c364f8..e131d0b2 100644 --- a/Makefile +++ b/Makefile @@ -203,7 +203,7 @@ kind-cluster: kind .PHONY: load-on-kind load-on-kind: kind-cluster ## Load the docker image into the kind cluster. - kind load docker-image ${IMG} -n ${KIND_CLUSTER_NAME} + $(LOCALBIN)/kind load docker-image ${IMG} -n ${KIND_CLUSTER_NAME} .PHONY: lint lint: From 38cac120cdbfa7546b427e23814cff4eb6e57611 Mon Sep 17 00:00:00 2001 From: Federico Paolinelli Date: Sat, 8 Jul 2023 12:08:23 +0300 Subject: [PATCH 2/3] Delete basic test Signed-off-by: Federico Paolinelli --- e2etests/tests/simple.go | 103 --------------------------------------- 1 file changed, 103 deletions(-) delete mode 100644 e2etests/tests/simple.go diff --git a/e2etests/tests/simple.go b/e2etests/tests/simple.go deleted file mode 100644 index b3f6b98e..00000000 --- a/e2etests/tests/simple.go +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier:Apache-2.0 - -package tests - -import ( - "context" - - "github.com/onsi/ginkgo/v2" - "go.universe.tf/e2etest/pkg/frr/container" - - frrk8sv1beta1 "github.com/metallb/frrk8s/api/v1beta1" - "github.com/metallb/frrk8stests/pkg/config" - "github.com/metallb/frrk8stests/pkg/dump" - "github.com/metallb/frrk8stests/pkg/infra" - "github.com/metallb/frrk8stests/pkg/k8s" - frrconfig "go.universe.tf/e2etest/pkg/frr/config" - "go.universe.tf/e2etest/pkg/ipfamily" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("FRR k8s", func() { - var cs clientset.Interface - var f *framework.Framework - - defer ginkgo.GinkgoRecover() - clientconfig, err := framework.LoadConfig() - framework.ExpectNoError(err) - updater, err := config.NewUpdater(clientconfig) - framework.ExpectNoError(err) - reporter := dump.NewK8sReporter(framework.TestContext.KubeConfig, k8s.FRRK8sNamespace) - - f = framework.NewDefaultFramework("bgpfrr") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - - ginkgo.AfterEach(func() { - if ginkgo.CurrentSpecReport().Failed() { - testName := ginkgo.CurrentSpecReport().LeafNodeText - dump.K8sInfo(testName, reporter) - dump.BGPInfo(testName, infra.FRRContainers, f.ClientSet, f) - } - }) - - ginkgo.BeforeEach(func() { - ginkgo.By("Clearing any previous configuration") - - for _, c := range infra.FRRContainers { - err := c.UpdateBGPConfigFile(frrconfig.Empty) - framework.ExpectNoError(err) - } - err := updater.Clean() - framework.ExpectNoError(err) - - cs = f.ClientSet - }) - - ginkgo.Context("Basic tests", func() { - ginkgo.It("establishes session with external frrs, IPV4", func() { - frrs := config.ContainersForVRF(infra.FRRContainers, "") - defaultVRFRouter := frrk8sv1beta1.Router{ - ASN: infra.FRRK8sASN, - Neighbors: config.NeighborsForContainers(frrs), - } - vrfFRRs := config.ContainersForVRF(infra.FRRContainers, infra.VRFName) - vrfRedRouter := frrk8sv1beta1.Router{ - ASN: infra.FRRK8sASNVRF, - Neighbors: config.NeighborsForContainers(vrfFRRs), - VRF: infra.VRFName, - } - - config := frrk8sv1beta1.FRRConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "default", - }, - Spec: frrk8sv1beta1.FRRConfigurationSpec{ - BGP: frrk8sv1beta1.BGPConfig{ - Routers: []frrk8sv1beta1.Router{ - defaultVRFRouter, - vrfRedRouter, - }, - }, - }, - } - - for _, c := range infra.FRRContainers { - err := container.PairWithNodes(cs, c, ipfamily.IPv4) - framework.ExpectNoError(err) - } - err := updater.Update(config) - framework.ExpectNoError(err) - - nodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - framework.ExpectNoError(err) - - for _, c := range infra.FRRContainers { - ValidateFRRPeeredWithNodes(nodes.Items, c, ipfamily.IPv4) - } - }) - }) -}) From 89d9c8176d5233f280b19bbca1aaae2a72b9a3ac Mon Sep 17 00:00:00 2001 From: Federico Paolinelli Date: Fri, 7 Jul 2023 10:16:00 +0200 Subject: [PATCH 3/3] E2E: add announcement tests We add basic announcement tests where all the nodes announce the same ips to all / some neighbors. Signed-off-by: Federico Paolinelli --- e2etests/go.mod | 2 +- e2etests/go.sum | 4 +- e2etests/go.work.sum | 7 + e2etests/pkg/config/from_containers.go | 49 ++++- e2etests/pkg/k8s/nodes.go | 21 ++ e2etests/pkg/routes/routes.go | 69 ++++++ e2etests/tests/advertisement.go | 289 +++++++++++++++++++++++++ e2etests/tests/validate.go | 30 +++ 8 files changed, 458 insertions(+), 13 deletions(-) create mode 100644 e2etests/pkg/k8s/nodes.go create mode 100644 e2etests/pkg/routes/routes.go create mode 100644 e2etests/tests/advertisement.go diff --git a/e2etests/go.mod b/e2etests/go.mod index f6a829ec..b01a4733 100644 --- a/e2etests/go.mod +++ b/e2etests/go.mod @@ -5,7 +5,7 @@ go 1.19 replace ( github.com/metallb/frrk8s => ../ github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 v2.11.0 - go.universe.tf/e2etest => github.com/metallb/metallb/e2etest v0.0.0-20230612085512-024c4f5b8e47 + go.universe.tf/e2etest => github.com/metallb/metallb/e2etest v0.0.0-20230704151713-ca4eed32d1f6 k8s.io/api => k8s.io/api v0.26.0 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.26.0 k8s.io/apimachinery => k8s.io/apimachinery v0.26.0 diff --git a/e2etests/go.sum b/e2etests/go.sum index ad795089..129d1ce2 100644 --- a/e2etests/go.sum +++ b/e2etests/go.sum @@ -226,8 +226,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/metallb/metallb/e2etest v0.0.0-20230612085512-024c4f5b8e47 h1:BVHjbnUbBC6Ka9MDMr66HYbLxTD/tI7ypPl1Bh2kV34= -github.com/metallb/metallb/e2etest v0.0.0-20230612085512-024c4f5b8e47/go.mod h1:pvpDmNSSuGzhAF32B9Rsf8cl+9XiAMq0fRMaWbJsWAo= +github.com/metallb/metallb/e2etest v0.0.0-20230704151713-ca4eed32d1f6 h1:PlT2uIiyi0jsbXax1wapWg0MjGoODcx5BODqdoXP40A= +github.com/metallb/metallb/e2etest v0.0.0-20230704151713-ca4eed32d1f6/go.mod h1:pvpDmNSSuGzhAF32B9Rsf8cl+9XiAMq0fRMaWbJsWAo= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= diff --git a/e2etests/go.work.sum b/e2etests/go.work.sum index a6b720e4..5c7da725 100644 --- a/e2etests/go.work.sum +++ b/e2etests/go.work.sum @@ -57,6 +57,7 @@ cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2H cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= @@ -87,6 +88,7 @@ cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LK cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= @@ -115,6 +117,7 @@ cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1r cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= @@ -260,6 +263,8 @@ github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHI github.com/mdlayher/ndp v0.0.0-20200602162440-17ab9e3e5567/go.mod h1:32w/5dDZWVSEOxyniAgKK4d7dHTuO6TCxWmUznQe3f8= github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= +github.com/metallb/metallb/e2etest v0.0.0-20230704151713-ca4eed32d1f6 h1:PlT2uIiyi0jsbXax1wapWg0MjGoODcx5BODqdoXP40A= +github.com/metallb/metallb/e2etest v0.0.0-20230704151713-ca4eed32d1f6/go.mod h1:pvpDmNSSuGzhAF32B9Rsf8cl+9XiAMq0fRMaWbJsWAo= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -331,6 +336,7 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= @@ -434,6 +440,7 @@ k8s.io/code-generator v0.26.0/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZme k8s.io/controller-manager v0.26.0/go.mod h1:GxUYtQDBE/RHh7AnZSZqwi2xBPIXlOaWsnDLflKGYrE= k8s.io/cri-api v0.26.0/go.mod h1:I5TGOn/ziMzqIcUvsYZzVE8xDAB1JBkvcwvR0yDreuw= k8s.io/csi-translation-lib v0.26.0/go.mod h1:zRKLRqER6rA8NCKQBhVIdkyDHKgNlu2BK1RKTHjcw+8= +k8s.io/dynamic-resource-allocation v0.26.0/go.mod h1:K+hO5A+QsSknRjlhfbUtvZVYUblOldvYyT51eGrZyWI= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= diff --git a/e2etests/pkg/config/from_containers.go b/e2etests/pkg/config/from_containers.go index 81ad430d..239ad86a 100644 --- a/e2etests/pkg/config/from_containers.go +++ b/e2etests/pkg/config/from_containers.go @@ -3,32 +3,61 @@ package config import ( + "net" + frrk8sv1beta1 "github.com/metallb/frrk8s/api/v1beta1" "go.universe.tf/e2etest/pkg/frr" frrcontainer "go.universe.tf/e2etest/pkg/frr/container" "go.universe.tf/e2etest/pkg/ipfamily" ) -// NeighborsForContainers returns a list of Neighbors for the given FRR containers. -func NeighborsForContainers(frrs []*frrcontainer.FRR, modify ...func(frr.Neighbor)) []frrk8sv1beta1.Neighbor { - res := make([]frrk8sv1beta1.Neighbor, 0) +type Peer struct { + IP string + Neigh frrk8sv1beta1.Neighbor + FRR frrcontainer.FRR +} + +// PeersForContainers returns two lists of Peers, one for v4 addresses and one for v6 addresses. +func PeersForContainers(frrs []*frrcontainer.FRR, ipFam ipfamily.Family, modify ...func(frr.Neighbor)) ([]Peer, []Peer) { + resV4 := make([]Peer, 0) + resV6 := make([]Peer, 0) for _, f := range frrs { - addresses := f.AddressesForFamily(ipfamily.IPv4) + addresses := f.AddressesForFamily(ipFam) ebgpMultihop := false if f.NeighborConfig.MultiHop && f.NeighborConfig.ASN != f.RouterConfig.ASN { ebgpMultihop = true } for _, address := range addresses { - neigh := frrk8sv1beta1.Neighbor{ - ASN: f.RouterConfig.ASN, - Address: address, - Port: f.RouterConfig.BGPPort, - EBGPMultiHop: ebgpMultihop, + peer := Peer{ + IP: address, + Neigh: frrk8sv1beta1.Neighbor{ + ASN: f.RouterConfig.ASN, + Address: address, + Port: f.RouterConfig.BGPPort, + EBGPMultiHop: ebgpMultihop, + }, + FRR: *f, + } + + if ipfamily.ForAddress(net.ParseIP(address)) == ipfamily.IPv4 { + resV4 = append(resV4, peer) + continue } - res = append(res, neigh) + resV6 = append(resV6, peer) } } + return resV4, resV6 +} + +func NeighborsFromPeers(peers []Peer, peers1 []Peer) []frrk8sv1beta1.Neighbor { + res := make([]frrk8sv1beta1.Neighbor, 0) + for _, p := range peers { + res = append(res, p.Neigh) + } + for _, p := range peers1 { + res = append(res, p.Neigh) + } return res } diff --git a/e2etests/pkg/k8s/nodes.go b/e2etests/pkg/k8s/nodes.go new file mode 100644 index 00000000..cbab0627 --- /dev/null +++ b/e2etests/pkg/k8s/nodes.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier:Apache-2.0 + +package k8s + +import ( + "context" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" +) + +// Nodes returns all nodes in the cluster. +func Nodes(cs clientset.Interface) ([]corev1.Node, error) { + nodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, errors.Wrap(err, "Failed to fetch frrk8s pods") + } + return nodes.Items, nil +} diff --git a/e2etests/pkg/routes/routes.go b/e2etests/pkg/routes/routes.go new file mode 100644 index 00000000..b6c1e0ef --- /dev/null +++ b/e2etests/pkg/routes/routes.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier:Apache-2.0 + +package routes + +import ( + "bytes" + "errors" + "fmt" + "net" + + "go.universe.tf/e2etest/pkg/frr" + frrcontainer "go.universe.tf/e2etest/pkg/frr/container" + "go.universe.tf/e2etest/pkg/ipfamily" + v1 "k8s.io/api/core/v1" +) + +// CheckNeighborHasPrefix tells if the given frr container has a route toward the given prefix +// via the set of node passed to this function. +func CheckNeighborHasPrefix(neighbor frrcontainer.FRR, prefix string, nodes []v1.Node) (bool, error) { + routesV4, routesV6, err := frr.Routes(neighbor) + if err != nil { + return false, err + } + + _, cidr, err := net.ParseCIDR(prefix) + if err != nil { + return false, err + } + + route, err := routeForCIDR(cidr, routesV4, routesV6) + var notFound RouteNotFoundError + if errors.As(err, ¬Found) { + return false, nil + } + if err != nil { + return false, err + } + + cidrFamily := ipfamily.ForCIDR(cidr) + err = frr.RoutesMatchNodes(nodes, route, cidrFamily, neighbor.RouterConfig.VRF) + if err != nil { + return false, nil + } + return true, nil +} + +func cidrsAreEqual(a, b *net.IPNet) bool { + return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask) +} + +type RouteNotFoundError string + +func (e RouteNotFoundError) Error() string { + return string(e) +} + +func routeForCIDR(cidr *net.IPNet, routesV4 map[string]frr.Route, routesV6 map[string]frr.Route) (frr.Route, error) { + for _, route := range routesV4 { + if cidrsAreEqual(route.Destination, cidr) { + return route, nil + } + } + for _, route := range routesV6 { + if cidrsAreEqual(route.Destination, cidr) { + return route, nil + } + } + return frr.Route{}, RouteNotFoundError(fmt.Sprintf("route %s not found", cidr)) +} diff --git a/e2etests/tests/advertisement.go b/e2etests/tests/advertisement.go new file mode 100644 index 00000000..e1b9aa7a --- /dev/null +++ b/e2etests/tests/advertisement.go @@ -0,0 +1,289 @@ +// SPDX-License-Identifier:Apache-2.0 + +package tests + +import ( + "github.com/onsi/ginkgo/v2" + "go.universe.tf/e2etest/pkg/frr/container" + + frrk8sv1beta1 "github.com/metallb/frrk8s/api/v1beta1" + "github.com/metallb/frrk8stests/pkg/config" + "github.com/metallb/frrk8stests/pkg/dump" + "github.com/metallb/frrk8stests/pkg/infra" + "github.com/metallb/frrk8stests/pkg/k8s" + frrconfig "go.universe.tf/e2etest/pkg/frr/config" + "go.universe.tf/e2etest/pkg/ipfamily" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = ginkgo.Describe("Advertisement", func() { + var cs clientset.Interface + var f *framework.Framework + + defer ginkgo.GinkgoRecover() + clientconfig, err := framework.LoadConfig() + framework.ExpectNoError(err) + updater, err := config.NewUpdater(clientconfig) + framework.ExpectNoError(err) + reporter := dump.NewK8sReporter(framework.TestContext.KubeConfig, k8s.FRRK8sNamespace) + + f = framework.NewDefaultFramework("bgpfrr") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + ginkgo.AfterEach(func() { + if ginkgo.CurrentSpecReport().Failed() { + testName := ginkgo.CurrentSpecReport().LeafNodeText + dump.K8sInfo(testName, reporter) + dump.BGPInfo(testName, infra.FRRContainers, f.ClientSet, f) + } + }) + + ginkgo.BeforeEach(func() { + ginkgo.By("Clearing any previous configuration") + + for _, c := range infra.FRRContainers { + err := c.UpdateBGPConfigFile(frrconfig.Empty) + framework.ExpectNoError(err) + } + err := updater.Clean() + framework.ExpectNoError(err) + + cs = f.ClientSet + }) + + ginkgo.Context("Advertising IPs", func() { + type params struct { + vrf string + ipFamily ipfamily.Family + myAsn uint32 + prefixes []string + modifyPeers func([]config.Peer, []config.Peer) + validate func([]config.Peer, []config.Peer, []v1.Node) + } + + ginkgo.DescribeTable("Works with external frrs", func(p params) { + frrs := config.ContainersForVRF(infra.FRRContainers, p.vrf) + peersV4, peersV6 := config.PeersForContainers(frrs, p.ipFamily) + p.modifyPeers(peersV4, peersV6) + neighbors := config.NeighborsFromPeers(peersV4, peersV6) + + config := frrk8sv1beta1.FRRConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: frrk8sv1beta1.FRRConfigurationSpec{ + BGP: frrk8sv1beta1.BGPConfig{ + Routers: []frrk8sv1beta1.Router{ + { + ASN: p.myAsn, + VRF: p.vrf, + Neighbors: neighbors, + Prefixes: p.prefixes, + }, + }, + }, + }, + } + + ginkgo.By("pairing with nodes") + for _, c := range frrs { + err := container.PairWithNodes(cs, c, p.ipFamily) + framework.ExpectNoError(err) + } + err := updater.Update(config) + framework.ExpectNoError(err) + + nodes, err := k8s.Nodes(cs) + framework.ExpectNoError(err) + + for _, c := range frrs { + ValidateFRRPeeredWithNodes(nodes, c, p.ipFamily) + } + + ginkgo.By("validating") + p.validate(peersV4, peersV6, nodes) + }, + ginkgo.Entry("IPV4 - Advertise with mode allowall", params{ + ipFamily: ipfamily.IPv4, + vrf: "", + myAsn: infra.FRRK8sASN, + prefixes: []string{"192.168.2.0/24", "192.169.2.0/24"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + for i := range ppV4 { + ppV4[i].Neigh.ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowAll + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + for _, p := range ppV4 { + ValidatePrefixesForNeighbor(p.FRR, nodes, "192.168.2.0/24", "192.169.2.0/24") + } + }, + }), + ginkgo.Entry("IPV4 - Advertise a subset of ips", params{ + ipFamily: ipfamily.IPv4, + vrf: "", + myAsn: infra.FRRK8sASN, + prefixes: []string{"192.168.2.0/24", "192.169.2.0/24"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + ppV4[0].Neigh.ToAdvertise.Allowed.Prefixes = []string{"192.168.2.0/24"} + for i := 1; i < len(ppV4); i++ { + ppV4[i].Neigh.ToAdvertise.Allowed.Prefixes = []string{"192.168.2.0/24", "192.169.2.0/24"} + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + ValidatePrefixesForNeighbor(ppV4[0].FRR, nodes, "192.168.2.0/24") + ValidateNeighborNoPrefixes(ppV4[0].FRR, nodes, "192.169.2.0/24") + for _, p := range ppV4[1:] { + ValidatePrefixesForNeighbor(p.FRR, nodes, "192.168.2.0/24", "192.169.2.0/24") + } + }, + }), + ginkgo.Entry("IPV6 - Advertise with mode allowall", params{ + ipFamily: ipfamily.IPv6, + vrf: "", + myAsn: infra.FRRK8sASN, + prefixes: []string{"fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + for i := range ppV6 { + ppV6[i].Neigh.ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowAll + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + for _, p := range ppV6 { + ValidatePrefixesForNeighbor(p.FRR, nodes, "fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64") + } + }, + }), + ginkgo.Entry("IPV6 - Advertise a subset of ips", params{ + ipFamily: ipfamily.IPv6, + vrf: "", + myAsn: infra.FRRK8sASN, + prefixes: []string{"fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + ppV6[0].Neigh.ToAdvertise.Allowed.Prefixes = []string{"fc00:f853:ccd:e799::/64"} + for i := 1; i < len(ppV6); i++ { + ppV6[i].Neigh.ToAdvertise.Allowed.Prefixes = []string{"fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"} + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + ValidatePrefixesForNeighbor(ppV6[0].FRR, nodes, "fc00:f853:ccd:e799::/64") + ValidateNeighborNoPrefixes(ppV6[0].FRR, nodes, "fc00:f853:ccd:e800::/64") + for _, p := range ppV6[1:] { + ValidatePrefixesForNeighbor(p.FRR, nodes, "fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64") + } + }, + }), + ginkgo.Entry("IPV4 - VRF - Advertise with mode allowall", params{ + ipFamily: ipfamily.IPv4, + vrf: infra.VRFName, + myAsn: infra.FRRK8sASNVRF, + prefixes: []string{"192.168.2.0/24", "192.169.2.0/24"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + for i := range ppV4 { + ppV4[i].Neigh.ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowAll + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + for _, p := range ppV4 { + ValidatePrefixesForNeighbor(p.FRR, nodes, "192.168.2.0/24", "192.169.2.0/24") + } + }, + }), + ginkgo.Entry("IPV4 - VRF - Advertise a subset of ips", params{ + ipFamily: ipfamily.IPv4, + vrf: infra.VRFName, + myAsn: infra.FRRK8sASNVRF, + prefixes: []string{"192.168.2.0/24", "192.169.2.0/24"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + ppV4[0].Neigh.ToAdvertise.Allowed.Prefixes = []string{"192.168.2.0/24"} + for i := 1; i < len(ppV4); i++ { + ppV4[i].Neigh.ToAdvertise.Allowed.Prefixes = []string{"192.168.2.0/24", "192.169.2.0/24"} + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + ValidatePrefixesForNeighbor(ppV4[0].FRR, nodes, "192.168.2.0/24") + ValidateNeighborNoPrefixes(ppV4[0].FRR, nodes, "192.169.2.0/24") + for _, p := range ppV4[1:] { + ValidatePrefixesForNeighbor(p.FRR, nodes, "192.168.2.0/24", "192.169.2.0/24") + } + }, + }), + ginkgo.Entry("IPV6 - Advertise a subset of ips", params{ + ipFamily: ipfamily.IPv6, + vrf: infra.VRFName, + myAsn: infra.FRRK8sASNVRF, + prefixes: []string{"fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + ppV6[0].Neigh.ToAdvertise.Allowed.Prefixes = []string{"fc00:f853:ccd:e799::/64"} + for i := 1; i < len(ppV6); i++ { + ppV6[i].Neigh.ToAdvertise.Allowed.Prefixes = []string{"fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"} + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + ValidatePrefixesForNeighbor(ppV6[0].FRR, nodes, "fc00:f853:ccd:e799::/64") + ValidateNeighborNoPrefixes(ppV6[0].FRR, nodes, "fc00:f853:ccd:e800::/64") + for _, p := range ppV6[1:] { + ValidatePrefixesForNeighbor(p.FRR, nodes, "fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64") + } + }, + }), + ginkgo.Entry("DUALSTACK - Advertise with mode allowall", params{ + ipFamily: ipfamily.DualStack, + vrf: "", + myAsn: infra.FRRK8sASN, + prefixes: []string{"192.168.2.0/24", "192.169.2.0/24", "fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + for i := range ppV4 { + ppV4[i].Neigh.ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowAll + } + for i := range ppV6 { + ppV6[i].Neigh.ToAdvertise.Allowed.Mode = frrk8sv1beta1.AllowAll + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + for _, p := range ppV4 { + ValidatePrefixesForNeighbor(p.FRR, nodes, "192.168.2.0/24", "192.169.2.0/24") + } + for _, p := range ppV6 { + ValidatePrefixesForNeighbor(p.FRR, nodes, "fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64") + } + }, + }), + ginkgo.Entry("DUALSTACK - Advertise a subset of ips", params{ + ipFamily: ipfamily.DualStack, + vrf: "", + myAsn: infra.FRRK8sASN, + prefixes: []string{"192.168.2.0/24", "192.169.2.0/24", "fc00:f853:ccd:e799::/64", "fc00:f853:ccd:e800::/64"}, + modifyPeers: func(ppV4 []config.Peer, ppV6 []config.Peer) { + ppV4[0].Neigh.ToAdvertise.Allowed.Prefixes = []string{"192.168.2.0/24"} + for i := 1; i < len(ppV4); i++ { + ppV4[i].Neigh.ToAdvertise.Allowed.Prefixes = []string{"192.169.2.0/24"} + } + ppV6[0].Neigh.ToAdvertise.Allowed.Prefixes = []string{"fc00:f853:ccd:e799::/64"} + for i := 1; i < len(ppV6); i++ { + ppV6[i].Neigh.ToAdvertise.Allowed.Prefixes = []string{"fc00:f853:ccd:e800::/64"} + } + }, + validate: func(ppV4 []config.Peer, ppV6 []config.Peer, nodes []v1.Node) { + ValidatePrefixesForNeighbor(ppV4[0].FRR, nodes, "192.168.2.0/24") + ValidateNeighborNoPrefixes(ppV4[0].FRR, nodes, "192.169.2.0/24") + for _, p := range ppV4[1:] { + ValidatePrefixesForNeighbor(p.FRR, nodes, "192.169.2.0/24") + ValidateNeighborNoPrefixes(p.FRR, nodes, "192.168.2.0/24") + } + ValidatePrefixesForNeighbor(ppV6[0].FRR, nodes, "fc00:f853:ccd:e799::/64") + ValidateNeighborNoPrefixes(ppV6[0].FRR, nodes, "fc00:f853:ccd:e800::/64") + for _, p := range ppV6[1:] { + ValidatePrefixesForNeighbor(p.FRR, nodes, "fc00:f853:ccd:e800::/64") + ValidateNeighborNoPrefixes(p.FRR, nodes, "fc00:f853:ccd:e799::/64") + } + }, + }), + ) + }) +}) diff --git a/e2etests/tests/validate.go b/e2etests/tests/validate.go index 1c47bbe3..f41a9b1f 100644 --- a/e2etests/tests/validate.go +++ b/e2etests/tests/validate.go @@ -6,12 +6,14 @@ import ( "fmt" "time" + "github.com/metallb/frrk8stests/pkg/routes" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.universe.tf/e2etest/pkg/frr" frrcontainer "go.universe.tf/e2etest/pkg/frr/container" "go.universe.tf/e2etest/pkg/ipfamily" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/kubernetes/test/e2e/framework" ) @@ -39,3 +41,31 @@ func ValidateFRRPeeredWithNodes(nodes []corev1.Node, c *frrcontainer.FRR, ipFami return nil }, 4*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) } + +func ValidatePrefixesForNeighbor(neigh frrcontainer.FRR, nodes []v1.Node, prefixes ...string) { + ginkgo.By(fmt.Sprintf("checking prefixes %v for %s", prefixes, neigh.Name)) + Eventually(func() error { + for _, prefix := range prefixes { + found, err := routes.CheckNeighborHasPrefix(neigh, prefix, nodes) + framework.ExpectNoError(err) + if !found { + fmt.Errorf("Neigh %s does not have prefix %s", neigh.Name, prefix) + } + } + return nil + }, time.Minute, time.Second).ShouldNot(HaveOccurred()) +} + +func ValidateNeighborNoPrefixes(neigh frrcontainer.FRR, nodes []v1.Node, prefixes ...string) { + ginkgo.By(fmt.Sprintf("checking prefixes %v not announced to %s", prefixes, neigh.Name)) + Eventually(func() error { + for _, prefix := range prefixes { + found, err := routes.CheckNeighborHasPrefix(neigh, prefix, nodes) + framework.ExpectNoError(err) + if found { + fmt.Errorf("Neigh %s has prefix %s", neigh.Name, prefix) + } + } + return nil + }, 5*time.Second, time.Second).ShouldNot(HaveOccurred()) +}