Merge branch 'master' into broadcom-support

cmd/internal/core/reconfigure-switch.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"slices"
 	"strconv"
@@ -23,7 +24,7 @@ func (c *Core) ReconfigureSwitch() {
 	for range t.C {
 		c.log.Info("trigger reconfiguration")
 		start := time.Now()
-		err := c.reconfigureSwitch(host)
+		s, err := c.reconfigureSwitch(host)
 		elapsed := time.Since(start)
 		c.log.Info("reconfiguration took", "elapsed", elapsed)
 
@@ -32,6 +33,7 @@ func (c *Core) ReconfigureSwitch() {
 		ns := elapsed.Nanoseconds()
 		nr := &models.V1SwitchNotifyRequest{
 			SyncDuration: &ns,
+			PortStates:   make(map[string]string),
 		}
 		if err != nil {
 			errStr := err.Error()
@@ -42,6 +44,30 @@ func (c *Core) ReconfigureSwitch() {
 			c.log.Info("reconfiguration succeeded")
 		}
 
+		// fill the port states of the switch
+		for _, n := range s.Nics {
+			if n == nil || n.Name == nil {
+				// lets log the whole nic because the name could be empty; lets hope there is some useful information
+				// in the nic
+				c.log.Error("could not check if link is up", "nic", n)
+				c.metrics.CountError("switch-reconfiguration")
+				continue
+			}
+			isup, err := isLinkUp(*n.Name)
+			if err != nil {
+				c.log.Error("could not check if link is up", "error", err, "nicname", *n.Name)
+				nr.PortStates[*n.Name] = models.V1SwitchNicActualUNKNOWN
+				c.metrics.CountError("switch-reconfiguration")
+				continue
+			}
+			if isup {
+				nr.PortStates[*n.Name] = models.V1SwitchNicActualUP
+			} else {
+				nr.PortStates[*n.Name] = models.V1SwitchNicActualDOWN
+			}
+
+		}
+
 		params.Body = nr
 		_, err = c.driver.SwitchOperations().NotifySwitch(params, nil)
 		if err != nil {
@@ -51,37 +77,37 @@ func (c *Core) ReconfigureSwitch() {
 	}
 }
 
-func (c *Core) reconfigureSwitch(switchName string) error {
+func (c *Core) reconfigureSwitch(switchName string) (*models.V1SwitchResponse, error) {
 	params := sw.NewFindSwitchParams()
 	params.ID = switchName
 	fsr, err := c.driver.SwitchOperations().FindSwitch(params, nil)
 	if err != nil {
-		return fmt.Errorf("could not fetch switch from metal-api: %w", err)
+		return nil, fmt.Errorf("could not fetch switch from metal-api: %w", err)
 	}
 
 	s := fsr.Payload
 	switchConfig, err := c.buildSwitcherConfig(s)
 	if err != nil {
-		return fmt.Errorf("could not build switcher config: %w", err)
+		return nil, fmt.Errorf("could not build switcher config: %w", err)
 	}
 
 	err = fillEth0Info(switchConfig, c.managementGateway)
 	if err != nil {
-		return fmt.Errorf("could not gather information about eth0 nic: %w", err)
+		return nil, fmt.Errorf("could not gather information about eth0 nic: %w", err)
 	}
 
 	c.log.Info("assembled new config for switch", "config", switchConfig)
 	if !c.enableReconfigureSwitch {
 		c.log.Debug("skip config application because of environment setting")
-		return nil
+		return s, nil
 	}
 
 	err = c.nos.Apply(switchConfig)
 	if err != nil {
-		return fmt.Errorf("could not apply switch config: %w", err)
+		return nil, fmt.Errorf("could not apply switch config: %w", err)
 	}
 
-	return nil
+	return s, nil
 }
 
 func (c *Core) buildSwitcherConfig(s *models.V1SwitchResponse) (*types.Conf, error) {
@@ -104,10 +130,18 @@ func (c *Core) buildSwitcherConfig(s *models.V1SwitchResponse) (*types.Conf, err
 		Unprovisioned: []string{},
 		Vrfs:          map[string]*types.Vrf{},
 		Firewalls:     map[string]*types.Firewall{},
+		DownPorts:     map[string]bool{},
 	}
 	p.BladePorts = c.additionalBridgePorts
 	for _, nic := range s.Nics {
 		port := *nic.Name
+
+		if isPortStatusEqual(models.V1SwitchNicActualDOWN, nic.Actual) {
+			if has := p.DownPorts[port]; !has {
+				p.DownPorts[port] = true
+			}
+		}
+
 		if slices.Contains(p.Underlay, port) {
 			continue
 		}
@@ -202,3 +236,20 @@ func fillEth0Info(c *types.Conf, gw string) error {
 	c.Ports.Eth0.Gateway = gw
 	return nil
 }
+
+// isLinkUp checks if the interface with the given name is up.
+// It returns a boolean indicating if the interface is up, and an error if there was a problem checking the interface.
+func isLinkUp(nicname string) (bool, error) {
+	nic, err := net.InterfaceByName(nicname)
+	if err != nil {
+		return false, fmt.Errorf("cannot query interface %q : %w", nicname, err)
+	}
+	return nic.Flags&net.FlagUp != 0, nil
+}
+
+func isPortStatusEqual(stat string, other *string) bool {
+	if other == nil {
+		return false
+	}
+	return strings.EqualFold(stat, *other)
+}

cmd/internal/core/reconfigure-switch_test.go

@@ -58,6 +58,7 @@ func TestBuildSwitcherConfig(t *testing.T) {
 		MetalCoreCIDR: "10.255.255.2/24",
 		ASN:           420000001,
 		Ports: types.Ports{
+			DownPorts:     map[string]bool{},
 			Underlay:      []string{"swp31", "swp32"},
 			Unprovisioned: []string{"swp1"},
 			Firewalls: map[string]*types.Firewall{

cmd/internal/switcher/cumulus/cumulus.go

@@ -29,12 +29,13 @@ func New(log *slog.Logger, frrTplFile, interfacesTplFile string) *Cumulus {
 }
 
 func (c *Cumulus) Apply(cfg *types.Conf) error {
-	err := c.interfacesApplier.Apply(cfg)
+	withoutDownPorts := cfg.NewWithoutDownPorts()
+	err := c.interfacesApplier.Apply(withoutDownPorts)
 	if err != nil {
 		return err
 	}
 
-	return c.frrApplier.Apply(cfg)
+	return c.frrApplier.Apply(withoutDownPorts)
 }
 
 func (c *Cumulus) IsInitialized() (initialized bool, err error) {

cmd/internal/switcher/sonic/db/configdb.go

@@ -18,6 +18,7 @@ const (
 	portTable       = "PORT"
 	adminStatus     = "admin_status"
 	adminStatusUp   = "up"
+	adminStatusDown = "down"
 	mtu             = "mtu"
 	fec             = "fec"
 	fecRS           = "rs"
@@ -231,8 +232,12 @@ func (d *ConfigDB) SetPortMtu(ctx context.Context, interfaceName string, val str
 	return d.c.HSet(ctx, key, Val{mtu: val})
 }
 
-func (d *ConfigDB) SetAdminStatusUp(ctx context.Context, interfaceName string) error {
+func (d *ConfigDB) SetAdminStatusUp(ctx context.Context, interfaceName string, up bool) error {
 	key := Key{portTable, interfaceName}
 
-	return d.c.HSet(ctx, key, Val{adminStatus: adminStatusUp})
+	status := adminStatusUp
+	if !up {
+		status = adminStatusDown
+	}
+	return d.c.HSet(ctx, key, Val{adminStatus: status})
 }

cmd/internal/switcher/sonic/redis/applier.go

@@ -49,19 +49,19 @@ func (a *Applier) Apply(cfg *types.Conf) error {
 	}
 
 	for _, interfaceName := range cfg.Ports.Underlay {
-		if err := a.configureUnderlayPort(interfaceName); err != nil {
+		if err := a.configureUnderlayPort(interfaceName, !cfg.Ports.DownPorts[interfaceName]); err != nil {
 			errs = append(errs, err)
 		}
 	}
 
 	for _, interfaceName := range cfg.Ports.Unprovisioned {
-		if err := a.configureUnprovisionedPort(interfaceName); err != nil {
+		if err := a.configureUnprovisionedPort(interfaceName, !cfg.Ports.DownPorts[interfaceName]); err != nil {
 			errs = append(errs, err)
 		}
 	}
 
 	for interfaceName := range cfg.Ports.Firewalls {
-		if err := a.configureFirewallPort(interfaceName); err != nil {
+		if err := a.configureFirewallPort(interfaceName, !cfg.Ports.DownPorts[interfaceName]); err != nil {
 			errs = append(errs, err)
 		}
 	}
@@ -71,7 +71,7 @@ func (a *Applier) Apply(cfg *types.Conf) error {
 			errs = append(errs, err)
 		}
 		for _, interfaceName := range vrf.Neighbors {
-			if err := a.configureVrfNeighbor(interfaceName, vrfName); err != nil {
+			if err := a.configureVrfNeighbor(interfaceName, vrfName, !cfg.Ports.DownPorts[interfaceName]); err != nil {
 				errs = append(errs, err)
 			}
 		}
@@ -115,7 +115,7 @@ func (a *Applier) refreshOidMaps() error {
 	return nil
 }
 
-func (a *Applier) configureUnprovisionedPort(interfaceName string) error {
+func (a *Applier) configureUnprovisionedPort(interfaceName string, isUp bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -124,14 +124,15 @@ func (a *Applier) configureUnprovisionedPort(interfaceName string) error {
 		return err
 	}
 
-	if err := a.ensurePortConfiguration(ctx, interfaceName, "9000", true); err != nil {
+	// unprovisioned ports should be up
+	if err := a.ensurePortConfiguration(ctx, interfaceName, "9000", true, isUp); err != nil {
 		return fmt.Errorf("failed to update Port info for interface %s: %w", interfaceName, err)
 	}
 
 	return a.ensureInterfaceIsVlanMember(ctx, interfaceName, "Vlan4000")
 }
 
-func (a *Applier) configureFirewallPort(interfaceName string) error {
+func (a *Applier) configureFirewallPort(interfaceName string, isUp bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -140,24 +141,26 @@ func (a *Applier) configureFirewallPort(interfaceName string) error {
 		return err
 	}
 
-	if err := a.ensurePortConfiguration(ctx, interfaceName, "9216", true); err != nil {
+	// a firewall port should always be up
+	if err := a.ensurePortConfiguration(ctx, interfaceName, "9216", true, isUp); err != nil {
 		return fmt.Errorf("failed to update Port info for interface %s: %w", interfaceName, err)
 	}
 
 	return a.ensureLinkLocalOnlyIsEnabled(ctx, interfaceName)
 }
 
-func (a *Applier) configureUnderlayPort(interfaceName string) error {
+func (a *Applier) configureUnderlayPort(interfaceName string, isUp bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	if err := a.ensurePortConfiguration(ctx, interfaceName, "9216", false); err != nil {
+	// underlay ports should be up
+	if err := a.ensurePortConfiguration(ctx, interfaceName, "9216", false, isUp); err != nil {
 		return fmt.Errorf("failed to update Port info for interface %s: %w", interfaceName, err)
 	}
 	return a.ensureLinkLocalOnlyIsEnabled(ctx, interfaceName)
 }
 
-func (a *Applier) configureVrfNeighbor(interfaceName, vrfName string) error {
+func (a *Applier) configureVrfNeighbor(interfaceName, vrfName string, isUp bool) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -171,7 +174,7 @@ func (a *Applier) configureVrfNeighbor(interfaceName, vrfName string) error {
 		return err
 	}
 
-	if err := a.ensurePortConfiguration(ctx, interfaceName, "9000", true); err != nil {
+	if err := a.ensurePortConfiguration(ctx, interfaceName, "9000", true, isUp); err != nil {
 		return fmt.Errorf("failed to update Port info for interface %s: %w", interfaceName, err)
 	}
 

cmd/internal/switcher/sonic/redis/port.go

@@ -7,7 +7,7 @@ import (
 	"github.com/avast/retry-go/v4"
 )
 
-func (a *Applier) ensurePortConfiguration(ctx context.Context, portName, mtu string, isFecRs bool) error {
+func (a *Applier) ensurePortConfiguration(ctx context.Context, portName, mtu string, isFecRs, isUp bool) error {
 	p, err := a.db.Config.GetPort(ctx, portName)
 	if err != nil {
 		return fmt.Errorf("could not retrieve port info for %s from redis: %w", portName, err)
@@ -29,9 +29,9 @@ func (a *Applier) ensurePortConfiguration(ctx context.Context, portName, mtu str
 		}
 	}
 
-	if !p.AdminStatus {
-		a.log.Debug("set admin status to", "port", portName, "admin_status", "up")
-		return a.db.Config.SetAdminStatusUp(ctx, portName)
+	if p.AdminStatus != isUp {
+		a.log.Debug("set admin status to", "port", portName, "admin_status_up", isUp)
+		return a.db.Config.SetAdminStatusUp(ctx, portName, isUp)
 	}
 
 	return nil

cmd/internal/switcher/sonic/sonic.go

@@ -146,6 +146,7 @@ func getPortsConfig(filepath string) (map[string]PortInfo, error) {
 	config := struct {
 		Ports map[string]PortInfo `json:"PORT"`
 	}{}
+	//nolint:musttag
 	err = json.Unmarshal(byteValue, &config)
 
 	return config.Ports, err

cmd/internal/switcher/templates/template_test.go

@@ -26,6 +26,13 @@ func TestInterfacesTemplate(t *testing.T) {
 	}
 }
 
+func TestInterfacesTemplateWithDownPorts(t *testing.T) {
+	c := readConf(t, path.Join("test_down_interfaces", "conf_with_downports.yaml"))
+	c = *c.NewWithoutDownPorts()
+	tpl := InterfacesTemplate("")
+	verifyTemplate(t, tpl, &c, path.Join("test_down_interfaces", "interfaces_with_downports"))
+}
+
 func TestCumulusFrrTemplate(t *testing.T) {
 	tests := listTestCases()
 	for i := range tests {
@@ -76,7 +83,7 @@ func TestCustomSonicFrrTemplate(t *testing.T) {
 func verifyTemplate(t *testing.T, tpl *template.Template, c *types.Conf, expectedFilename string) {
 	actual := renderToString(t, tpl, c)
 	expected := readExpected(t, expectedFilename)
-	require.Equal(t, expected, actual, "Wanted: %s, Got: %s", expected, actual)
+	require.Equal(t, expected, actual, "Wanted: %s\nGot: %s", expected, actual)
 }
 
 func renderToString(t *testing.T, tpl *template.Template, c *types.Conf) string {

cmd/internal/switcher/templates/test_down_interfaces/conf_with_downports.yaml

@@ -0,0 +1,26 @@
+---
+name: leaf01
+loglevel: warnings
+loopback: 10.0.0.10
+asn: 4200000010
+metalcorecidr: 10.255.255.2/24
+ports:
+  vrfs:
+    vrf1:
+      vni: 1
+      vlanid: 2
+      neighbors:
+        - swp3
+      cidrs:
+        - 10.255.255.1/28
+  eth0:
+    addresscidr: 192.168.0.11
+    gateway: 192.168.0.254
+  underlay:
+    - swp31
+    - swp32
+  unprovisioned:
+    - swp1
+    - swp2
+  downports:
+    swp3: True