Merge pull request #507 from TrekkieCoder/main

PR - Overall fixes

Dockerfile

@@ -45,7 +45,6 @@ RUN mkdir -p /opt/loxilb && \
     git clone --recurse-submodules https://github.com/loxilb-io/loxilb  /root/loxilb-io/loxilb/ && \
     cd /root/loxilb-io/loxilb/ && go get . && make && \
     cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \
-    cp loxilb-ebpf/utils/loxilb_dp_tool /usr/local/sbin/loxilb_dp_tool && \
     cp api/certification/* /opt/loxilb/cert/ && cd - && \
     cp /root/loxilb-io/loxilb/loxilb-ebpf/kernel/loxilb_dp_debug  /usr/local/sbin/loxilb_dp_debug && \
     cp /root/loxilb-io/loxilb/loxilb /usr/local/sbin/loxilb && \

api/api.go

@@ -77,6 +77,8 @@ func RunAPIServer() {
 		if e := recover(); e != nil {
 			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
 		}
+		handler.ApiHooks.NetHandlePanic()
+		os.Exit(1)
 	}()
 
 	if ApiShutOk == nil {

api/loxinlp/nlp.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/exec"
 	"regexp"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"syscall"
@@ -1378,8 +1379,18 @@ func RUWorker(ch chan nlp.RouteUpdate, f chan struct{}) {
 	}
 }
 
-func NLWorker(nNl *NlH, bgpPeerMode bool, ch chan bool) {
+func NLWorker(nNl *NlH, bgpPeerMode bool, ch chan bool, wch chan bool) {
 	ch <- true
+	<-wch
+
+	defer func() {
+		if e := recover(); e != nil {
+			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
+		}
+		hooks.NetHandlePanic()
+		os.Exit(1)
+	}()
+
 	if bgpPeerMode {
 		for { /* Single thread for reading route NL msgs in below order */
 			RUWorker(nNl.FromRUCh, nNl.FromRUDone)
@@ -1426,6 +1437,18 @@ func NlpGet(ch chan bool) int {
 		ret = -1
 	}
 
+	for _, link := range links {
+
+		if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) {
+			continue
+		}
+
+		ret = ModLink(link, true)
+		if ret == -1 {
+			continue
+		}
+	}
+
 	for _, link := range links {
 
 		if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) {
@@ -1446,11 +1469,6 @@ func NlpGet(ch chan bool) int {
 			continue
 		}
 
-		ret = ModLink(link, true)
-		if ret == -1 {
-			continue
-		}
-
 		/* Get FDBs */
 		_, ok := link.(*nlp.Vxlan)
 		if link.Attrs().MasterIndex > 0 || ok {
@@ -1580,6 +1598,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH {
 	nNl.BlackList = blackList
 	nNl.BLRgx = regexp.MustCompile(blackList)
 	checkInit := make(chan bool)
+	waitInit := make(chan bool)
 
 	if bgpPeerMode {
 		nNl.FromRUCh = make(chan nlp.RouteUpdate, cmn.RuWorkQLen)
@@ -1590,7 +1609,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH {
 			tk.LogIt(tk.LogInfo, "[NLP] Route msgs subscribed\n")
 		}
 
-		go NLWorker(nNl, bgpPeerMode, checkInit)
+		go NLWorker(nNl, bgpPeerMode, checkInit, waitInit)
 		<-checkInit
 		return nNl
 	}
@@ -1605,7 +1624,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH {
 	nNl.FromRUCh = make(chan nlp.RouteUpdate, cmn.RuWorkQLen)
 	nNl.IMap = make(map[string]Intf)
 
-	go NLWorker(nNl, bgpPeerMode, checkInit)
+	go NLWorker(nNl, bgpPeerMode, checkInit, waitInit)
 	<-checkInit
 
 	err := nlp.LinkSubscribe(nNl.FromLUCh, nNl.FromLUDone)
@@ -1637,6 +1656,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH {
 
 	go NlpGet(checkInit)
 	done := <-checkInit
+	waitInit <- true
 
 	go LbSessionGet(done)
 

common/common.go

@@ -65,11 +65,11 @@ const (
 
 const (
 	// AuWorkqLen - Address worker channel depth
-	AuWorkqLen = 1024
+	AuWorkqLen = 2048
 	// LuWorkQLen - Link worker channel depth
-	LuWorkQLen = 1024
+	LuWorkQLen = 2048
 	// NuWorkQLen - Neigh worker channel depth
-	NuWorkQLen = 1024
+	NuWorkQLen = 2048
 	// RuWorkQLen - Route worker channel depth
 	RuWorkQLen = 40827
 )
@@ -833,4 +833,5 @@ type NetHookInterface interface {
 	NetGoBGPNeighAdd(nm *GoBGPNeighMod) (int, error)
 	NetGoBGPNeighDel(nm *GoBGPNeighMod) (int, error)
 	NetGoBGPGCAdd(gc *GoBGPGlobalConfig) (int, error)
+	NetHandlePanic()
 }

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/go-openapi/validate v0.22.0
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/loxilb-io/ipvs v0.1.0
-	github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f
+	github.com/loxilb-io/loxilib v0.8.9-0.20240124072521-f37fbddfb4e8
 	github.com/osrg/gobgp/v3 v3.5.0
 	github.com/prometheus-community/pro-bing v0.1.0
 	github.com/prometheus/client_model v0.3.0

go.sum

@@ -262,6 +262,8 @@ github.com/loxilb-io/ipvs v0.1.0 h1:TpTkwh5CLgJ7YW86rvWyqJPEpQFqs2TNbRG/IECeq+w=
 github.com/loxilb-io/ipvs v0.1.0/go.mod h1:EKjimnzyVL9AXMMNfPWeokxF1uNeuDrEGF5gPFMdmIo=
 github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f h1:q745LqB2G97bOpf6MCa34VyNJEuvVx4rTFVut3kz9oo=
 github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f/go.mod h1:LoQCxBz+N0fO9rGwRmPHrQPHol/jUf4MNpph63Cydkg=
+github.com/loxilb-io/loxilib v0.8.9-0.20240124072521-f37fbddfb4e8 h1:PYkgIX2fv1UluRR82SAot3O5YnwwCbcWc7VULvPv3QM=
+github.com/loxilb-io/loxilib v0.8.9-0.20240124072521-f37fbddfb4e8/go.mod h1:LoQCxBz+N0fO9rGwRmPHrQPHol/jUf4MNpph63Cydkg=
 github.com/loxilb-io/sctp v0.0.0-20230519081703-6d1baec82fd4 h1:oDc2lsbfuQEcVP3k+Pw4v6Xdm3t4M9vBc1Y9egszv6g=
 github.com/loxilb-io/sctp v0.0.0-20230519081703-6d1baec82fd4/go.mod h1:1a6hv8ISVQhnW5IVpW9o+OL6BAFlWiVpC0O4d19g+wQ=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

loxinet/apiclient.go

@@ -629,3 +629,8 @@ func (na *NetAPIStruct) NetGoBGPGCAdd(param *cmn.GoBGPGlobalConfig) (int, error)
 	return 0, errors.New("loxilb BGP mode is disabled")
 
 }
+
+// NetHandlePanic - Handle panics
+func (na *NetAPIStruct) NetHandlePanic() {
+	mh.dp.DpHooks.DpEbpfUnInit()
+}

loxinet/dpbroker.go

@@ -19,6 +19,7 @@ package loxinet
 import (
 	"fmt"
 	"net"
+	"os"
 	"runtime/debug"
 	"sync"
 	"time"
@@ -411,6 +412,7 @@ type DpHookInterface interface {
 	DpCtGetAsync()
 	DpGetLock()
 	DpRelLock()
+	DpEbpfUnInit()
 }
 
 // DpPeer - Remote DP Peer information
@@ -786,6 +788,10 @@ func DpWorker(dp *DpH, f chan int, ch chan interface{}) {
 		if e := recover(); e != nil {
 			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
 		}
+		if mh.dp != nil {
+			mh.dp.DpHooks.DpEbpfUnInit()
+		}
+		os.Exit(1)
 	}()
 	for {
 		for n := 0; n < DpWorkQLen; n++ {

loxinet/dpebpf_linux.go

@@ -45,6 +45,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"runtime/debug"
 	"strings"
 	"sync"
@@ -157,6 +158,10 @@ func dpEbpfTicker() {
 		if e := recover(); e != nil {
 			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
 		}
+		if mh.dp != nil {
+			mh.dp.DpHooks.DpEbpfUnInit()
+		}
+		os.Exit(1)
 	}()
 
 	tbls := []int{int(C.LL_DP_RTV4_STATS_MAP),
@@ -312,7 +317,7 @@ func (e *DpEbpfH) DpEbpfUnInit() {
 		e.ToFinCh[i] <- 1
 	}
 
-	tk.LogIt(tk.LogInfo, "ebpf uninit \n")
+	tk.LogIt(tk.LogInfo, "ebpf uninit : %s\n", debug.Stack())
 
 	// Make sure to unload eBPF programs
 	ifList, err := net.Interfaces()
@@ -1918,6 +1923,10 @@ func dpMapNotifierWorker(f chan int, ch chan interface{}) {
 		if e := recover(); e != nil {
 			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
 		}
+		if mh.dp != nil {
+			mh.dp.DpHooks.DpEbpfUnInit()
+		}
+		os.Exit(1)
 	}()
 
 	for {

loxinet/layer2.go

@@ -291,7 +291,7 @@ func (l2 *L2H) L2FdbDel(key FdbKey) (int, error) {
 
 // FdbTicker - Ticker routine for a fwd entry
 func (l2 *L2H) FdbTicker(f *FdbEnt) {
-	if time.Now().Sub(f.stime) > FdbGts {
+	if time.Since(f.stime) > FdbGts {
 		// This scans for inconsistencies in a fdb
 		// 1. Do garbage cleaning if underlying oif or vlan is not valid anymore
 		// 2. If FDB is a TunFDB, we need to make sure NH is reachable
@@ -304,6 +304,8 @@ func (l2 *L2H) FdbTicker(f *FdbEnt) {
 				f.unReach = unRch
 				f.DP(DpCreate)
 			}
+		} else if f.Sync != 0 {
+			f.DP(DpCreate)
 		}
 		f.stime = time.Now()
 	}
@@ -385,6 +387,10 @@ func (f *FdbEnt) DP(work DpWorkT) int {
 		l2Wq.Tagged = 0
 	} else {
 		l2Wq.Tagged = 1
+		if f.Port.SInfo.PortReal == nil {
+			f.Sync = DpUknownErr
+			return -1
+		}
 		l2Wq.PortNum = f.Port.SInfo.PortReal.PortNo
 	}
 	mh.dp.ToDpCh <- l2Wq
@@ -396,6 +402,7 @@ func (f *FdbEnt) DP(work DpWorkT) int {
 
 		if f.Port.SInfo.PortReal == nil ||
 			f.FdbTun.ep == nil {
+			f.Sync = DpUknownErr
 			return -1
 		}
 

loxinet/loxinet.go

@@ -129,6 +129,17 @@ func (mh *loxiNetH) ParamGet(param *cmn.ParamMod) (int, error) {
 
 // loxiNetTicker - this ticker routine runs every LOXINET_TIVAL seconds
 func loxiNetTicker(bgpPeerMode bool) {
+
+	defer func() {
+		if e := recover(); e != nil {
+			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
+		}
+		if mh.dp != nil {
+			mh.dp.DpHooks.DpEbpfUnInit()
+		}
+		os.Exit(1)
+	}()
+
 	for {
 		select {
 		case <-mh.tDone:
@@ -182,13 +193,6 @@ func loxiNetInit() {
 	logLevel := LogString2Level(opts.Opts.LogLevel)
 	mh.logger = tk.LogItInit(logfile, logLevel, true)
 
-	// Stack trace logger
-	defer func() {
-		if e := recover(); e != nil {
-			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
-		}
-	}()
-
 	// It is important to make sure loxilb's eBPF filesystem
 	// is in place and mounted to make sure maps are pinned properly
 	if !FileExists(BpfFsCheckFile) {
@@ -312,6 +316,16 @@ func loxiNetInit() {
 
 // loxiNetRun - This routine will not return
 func loxiNetRun() {
+	// Stack trace logger
+	defer func() {
+		if e := recover(); e != nil {
+			tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
+		}
+		if mh.dp != nil {
+			mh.dp.DpHooks.DpEbpfUnInit()
+		}
+		os.Exit(1)
+	}()
 	mh.wg.Wait()
 }
 

loxinet/port.go

@@ -914,7 +914,8 @@ func (p *Port) DP(work DpWorkT) int {
 		ipts.Qfi = 0
 		ipts.TTeID = 0
 
-		mh.dp.ToDpCh <- ipts
+		//mh.dp.ToDpCh <- ipts
+		DpWorkSingle(mh.dp, ipts)
 		return 0
 	}
 
@@ -944,7 +945,8 @@ func (p *Port) DP(work DpWorkT) int {
 		rmWq.TunType = DpTunVxlan
 		rmWq.BD = p.L2.Vid
 
-		mh.dp.ToDpCh <- rmWq
+		//mh.dp.ToDpCh <- rmWq
+		DpWorkSingle(mh.dp, rmWq)
 
 		return 0
 	}
@@ -961,7 +963,8 @@ func (p *Port) DP(work DpWorkT) int {
 		pWq.IngVlan = p.L2.Vid
 		pWq.SetBD = p.L2.Vid
 		pWq.SetZoneNum = zoneNum
-		mh.dp.ToDpCh <- pWq
+		//mh.dp.ToDpCh <- pWq
+		DpWorkSingle(mh.dp, pWq)
 
 		return 0
 	}
@@ -981,7 +984,8 @@ func (p *Port) DP(work DpWorkT) int {
 			pWq.SetPol = p.SInfo.PortPolNum
 			pWq.SetMirr = p.SInfo.PortMirNum
 
-			mh.dp.ToDpCh <- pWq
+			//mh.dp.ToDpCh <- pWq
+			DpWorkSingle(mh.dp, pWq)
 		}
 		return 0
 	}
@@ -1039,7 +1043,8 @@ func (p *Port) DP(work DpWorkT) int {
 	}
 
 	// TODO - Need to unload eBPF when port properties change
-	mh.dp.ToDpCh <- pWq
+	//mh.dp.ToDpCh <- pWq
+	DpWorkSingle(mh.dp, pWq)
 
 	return 0
 }

loxinet/xsync_server.go

@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	"net/rpc"
+	"os"
 	"runtime/debug"
 
 	opts "github.com/loxilb-io/loxilb/options"
@@ -218,6 +219,10 @@ func LoxiXsyncMain(mode string) {
 			if mh.logger != nil {
 				tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack())
 			}
+			if mh.dp != nil {
+				mh.dp.DpHooks.DpEbpfUnInit()
+			}
+			os.Exit(1)
 		}
 	}()
 	if mode == "netrpc" {