Skip to content

Commit

Permalink
Remove assumption that internal == tap (#289)
Browse files Browse the repository at this point in the history 
* Remove assumption that internal == tap

* fix interface

* fix
  • Loading branch information
@DanG100
DanG100 authored Oct 5, 2023
1 parent a0e5479 commit 52190cc
Show file tree
Hide file tree
Showing 11 changed files with 368 additions and 288 deletions.
2 changes: 1 addition & 1 deletion dataplane/forwarding/fwdport/ports/kernel.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (p *kernelPort) process() {
return
default:
d, _, err := p.handle.ReadPacketData()
if err == afpacket.ErrTimeout { // Don't log this error as it is very spammy.
if err == afpacket.ErrTimeout || err == afpacket.ErrPoll { // Don't log this error as it is very spammy.
continue
}
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions dataplane/handlers/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,7 @@ func (ni *Interface) setupPorts(ctx context.Context) error {
Src: &dpb.CreatePortRequest_KernelDev{
KernelDev: i.Name,
},
Location: dpb.PortLocation_PORT_LOCATION_EXTERNAL,
})
if err != nil {
return fmt.Errorf("failed to create kernel interface %q: %w", i.Name, err)
Expand All @@ -588,6 +589,7 @@ func (ni *Interface) setupPorts(ctx context.Context) error {
KernelDev: ni.externalToInternalPort[i.Name],
},
ExternalPort: i.Name,
Location: dpb.PortLocation_PORT_LOCATION_INTERNAL,
})
if err != nil {
return fmt.Errorf("failed to create tap interface %q: %w", ni.externalToInternalPort[i.Name], err)
Expand Down
24 changes: 12 additions & 12 deletions dataplane/internal/engine/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,12 +284,12 @@ func (e *Engine) ID() string {

func (e *Engine) CreatePort(ctx context.Context, req *dpb.CreatePortRequest) (*dpb.CreatePortResponse, error) {
var err error
switch req.Type {
case fwdpb.PortType_PORT_TYPE_KERNEL:
err = e.CreateExternalPort(ctx, req.GetId(), req.GetKernelDev())
case fwdpb.PortType_PORT_TYPE_TAP:
err = e.CreateInternalPort(ctx, req.GetId(), req.GetKernelDev(), req.GetExternalPort())
case fwdpb.PortType_PORT_TYPE_CPU_PORT:
switch req.Location {
case dpb.PortLocation_PORT_LOCATION_EXTERNAL:
err = e.CreateExternalPort(ctx, req.GetType(), req.GetId(), req.GetKernelDev())
case dpb.PortLocation_PORT_LOCATION_INTERNAL:
err = e.CreateInternalPort(ctx, req.GetType(), req.GetId(), req.GetKernelDev(), req.GetExternalPort())
case dpb.PortLocation_PORT_LOCATION_CPU:
e.cpuPortID = req.GetId()
req := &fwdpb.PortCreateRequest{
ContextId: &fwdpb.ContextId{Id: e.id},
Expand All @@ -303,7 +303,7 @@ func (e *Engine) CreatePort(ctx context.Context, req *dpb.CreatePortRequest) (*d
}
_, err = e.PortCreate(ctx, req)
default:
return nil, fmt.Errorf("invalid port type")
return nil, fmt.Errorf("invalid port location: %v", req.Location)
}
return &dpb.CreatePortResponse{}, err
}
Expand Down Expand Up @@ -923,9 +923,9 @@ func (e *Engine) UpdatePortSrcMAC(ctx context.Context, portID string, mac []byte
}

// CreateExternalPort creates an external port (connected to other devices).
func (e *Engine) CreateExternalPort(ctx context.Context, id, devName string) error {
func (e *Engine) CreateExternalPort(ctx context.Context, t fwdpb.PortType, id, devName string) error {
log.Infof("added external id %s, dev %s", id, devName)
nid, err := createKernelPort(ctx, e.id, e.Server, id, devName)
nid, err := createPortAndEntries(ctx, e.id, e.Server, t, id, devName)
if err != nil {
return err
}
Expand Down Expand Up @@ -1003,9 +1003,9 @@ func (e *Engine) CreateExternalPort(ctx context.Context, id, devName string) err
}

// CreateInternalPort creates an local (ie TAP) port for the given linux device name.
func (e *Engine) CreateInternalPort(ctx context.Context, id, devName, externalID string) error {
func (e *Engine) CreateInternalPort(ctx context.Context, t fwdpb.PortType, id, devName, externalID string) error {
log.Infof("added internal id %s, dev %s, external %s", id, devName, externalID)
nid, err := createTapPort(ctx, e.id, e.Server, id, devName)
nid, err := createPortAndEntries(ctx, e.id, e.Server, t, id, devName)
if err != nil {
return err
}
Expand Down Expand Up @@ -1165,7 +1165,7 @@ func (e *Engine) AddInterface(ctx context.Context, req *dpb.AddInterfaceRequest)
}
case dpb.InterfaceType_INTERFACE_TYPE_LOOPBACK: // TODO: this may need to handled differently if multiple loopbacks are created.
portID := fmt.Sprintf("%s-port", req.GetId())
if err := e.CreateExternalPort(ctx, portID, "lo"); err != nil {
if err := e.CreateExternalPort(ctx, fwdpb.PortType_PORT_TYPE_KERNEL, portID, "lo"); err != nil {
return nil, err
}
e.ifaceToPort[req.GetId()] = portID
Expand Down
42 changes: 8 additions & 34 deletions dataplane/internal/engine/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,12 +281,12 @@ func createLayer3PuntTable(ctx context.Context, ctxID string, c fwdpb.Forwarding
return nil
}

// createKernelPort creates a port using the "Kernel" dataplane type (socket API).
func createKernelPort(ctx context.Context, ctxID string, c fwdpb.ForwardingServer, id, devName string) (uint64, error) {
// createPortAndEntries creates a port and sets up the punt rules
func createPortAndEntries(ctx context.Context, ctxID string, c fwdpb.ForwardingServer, t fwdpb.PortType, id, devName string) (uint64, error) {
port := &fwdpb.PortCreateRequest{
ContextId: &fwdpb.ContextId{Id: ctxID},
Port: &fwdpb.PortDesc{
PortType: fwdpb.PortType_PORT_TYPE_KERNEL,
PortType: t,
PortId: &fwdpb.PortId{
ObjectId: &fwdpb.ObjectId{Id: id},
},
Expand All @@ -297,37 +297,12 @@ func createKernelPort(ctx context.Context, ctxID string, c fwdpb.ForwardingServe
},
},
}
portID, err := c.PortCreate(ctx, port)
if err != nil {
return 0, err
}
if err := addLayer2PuntRule(ctx, ctxID, c, portID.GetObjectIndex().GetIndex(), etherBroadcast, etherBroadcastMask); err != nil {
return 0, err
}
if err := addLayer2PuntRule(ctx, ctxID, c, portID.GetObjectIndex().GetIndex(), etherMulticast, etherMulticastMask); err != nil {
return 0, err
}
if err := addLayer2PuntRule(ctx, ctxID, c, portID.GetObjectIndex().GetIndex(), etherIPV6Multi, etherIPV6MultiMask); err != nil {
return 0, err
}
return portID.GetObjectIndex().GetIndex(), nil
}

// createKernelPort creates a port using the "TAP" dataplane type (tap file API) and returns the fd to read/write from.
func createTapPort(ctx context.Context, ctxID string, c fwdpb.ForwardingServer, id, devName string) (uint64, error) {
port := &fwdpb.PortCreateRequest{
ContextId: &fwdpb.ContextId{Id: ctxID},
Port: &fwdpb.PortDesc{
PortType: fwdpb.PortType_PORT_TYPE_TAP,
PortId: &fwdpb.PortId{
ObjectId: &fwdpb.ObjectId{Id: id},
},
Port: &fwdpb.PortDesc_Tap{
Tap: &fwdpb.TAPPortDesc{
DeviceName: devName,
},
if t == fwdpb.PortType_PORT_TYPE_TAP {
port.Port.Port = &fwdpb.PortDesc_Tap{
Tap: &fwdpb.TAPPortDesc{
DeviceName: devName,
},
},
}
}
portID, err := c.PortCreate(ctx, port)
if err != nil {
Expand All @@ -342,6 +317,5 @@ func createTapPort(ctx context.Context, ctxID string, c fwdpb.ForwardingServer,
if err := addLayer2PuntRule(ctx, ctxID, c, portID.GetObjectIndex().GetIndex(), etherIPV6Multi, etherIPV6MultiMask); err != nil {
return 0, err
}

return portID.GetObjectIndex().GetIndex(), nil
}
3 changes: 1 addition & 2 deletions dataplane/standalone/entrypoint.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,10 +178,9 @@ std::unique_ptr<lemming::dataplane::sai::Wred::Stub> wred;

std::unique_ptr<lemming::dataplane::sai::Entrypoint::Stub> entry;

// TODO(dgrau): implement this without using gRPC.
sai_status_t sai_api_initialize(
_In_ uint64_t flags, _In_ const sai_service_method_table_t *services) {
FLAGS_log_dir = "/var/log/syncd";
FLAGS_log_dir = "/var/log";
google::InitGoogleLogging("lucius");
google::InstallFailureSignalHandler();

Expand Down
24 changes: 18 additions & 6 deletions dataplane/standalone/saiserver/ports.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,13 @@ func (port *port) CreatePort(ctx context.Context, _ *saipb.CreatePortRequest) (*
QosDscpToForwardingClassMap: proto.Uint64(0),
QosMplsExpToForwardingClassMap: proto.Uint64(0),
IpsecPort: proto.Uint64(0),
SupportedSpeed: []uint32{1024},
OperSpeed: proto.Uint32(1024),
SupportedSpeed: []uint32{1000, 10000, 40000},
OperSpeed: proto.Uint32(40000),
SupportedFecMode: []saipb.PortFecMode{saipb.PortFecMode_PORT_FEC_MODE_NONE},
NumberOfIngressPriorityGroups: proto.Uint32(0),
QosMaximumHeadroomSize: proto.Uint32(0),
AdminState: proto.Bool(true),
AutoNegMode: proto.Bool(false),
AutoNegMode: proto.Bool(true),
Mtu: proto.Uint32(1514),
}

Expand All @@ -139,6 +139,7 @@ func (port *port) CreatePort(ctx context.Context, _ *saipb.CreatePortRequest) (*
Src: &dpb.CreatePortRequest_KernelDev{
KernelDev: dev,
},
Location: dpb.PortLocation_PORT_LOCATION_EXTERNAL,
})
if err != nil {
return nil, err
Expand All @@ -155,8 +156,9 @@ func (port *port) createCPUPort(ctx context.Context) (uint64, error) {
id := port.mgr.NextID()

_, err := port.dataplane.CreatePort(ctx, &dpb.CreatePortRequest{
Id: fmt.Sprint(id),
Type: fwdpb.PortType_PORT_TYPE_CPU_PORT,
Id: fmt.Sprint(id),
Type: fwdpb.PortType_PORT_TYPE_CPU_PORT,
Location: dpb.PortLocation_PORT_LOCATION_CPU,
})
if err != nil {
return 0, err
Expand Down Expand Up @@ -255,6 +257,11 @@ func (port *port) SetPortAttribute(ctx context.Context, req *saipb.SetPortAttrib
return &saipb.SetPortAttributeResponse{}, nil
}

func (port *port) Reset() {
port.portToEth = make(map[uint64]string)
port.nextEth = 1
}

func newHostif(mgr *attrmgr.AttrMgr, dataplane portDataplaneAPI, s *grpc.Server) *hostif {
p := &hostif{
mgr: mgr,
Expand All @@ -280,10 +287,11 @@ func (hostif *hostif) CreateHostif(ctx context.Context, req *saipb.CreateHostifR
case saipb.HostifType_HOSTIF_TYPE_NETDEV:
portReq := &dpb.CreatePortRequest{
Id: fmt.Sprint(id),
Type: fwdpb.PortType_PORT_TYPE_TAP,
Type: fwdpb.PortType_PORT_TYPE_KERNEL,
Src: &dpb.CreatePortRequest_KernelDev{
KernelDev: string(req.GetName()),
},
Location: dpb.PortLocation_PORT_LOCATION_INTERNAL,
}
attrReq := &saipb.GetPortAttributeRequest{Oid: req.GetObjId(), AttrType: []saipb.PortAttr{saipb.PortAttr_PORT_ATTR_OPER_STATUS}}
p := &saipb.GetPortAttributeResponse{}
Expand All @@ -298,6 +306,10 @@ func (hostif *hostif) CreateHostif(ctx context.Context, req *saipb.CreateHostifR
return nil, err
}
}
attr := &saipb.HostifAttribute{
OperStatus: proto.Bool(true),
}
hostif.mgr.StoreAttributes(id, attr)
default:
return nil, status.Errorf(codes.InvalidArgument, "unknown type %v", req.GetType())
}
Expand Down
17 changes: 9 additions & 8 deletions dataplane/standalone/saiserver/ports_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,13 +94,13 @@ func TestCreatePort(t *testing.T) {
QosDscpToForwardingClassMap: proto.Uint64(0),
QosMplsExpToForwardingClassMap: proto.Uint64(0),
IpsecPort: proto.Uint64(0),
SupportedSpeed: []uint32{1024},
OperSpeed: proto.Uint32(1024),
SupportedSpeed: []uint32{1000, 10000, 40000},
OperSpeed: proto.Uint32(40000),
SupportedFecMode: []saipb.PortFecMode{saipb.PortFecMode_PORT_FEC_MODE_NONE},
NumberOfIngressPriorityGroups: proto.Uint32(0),
QosMaximumHeadroomSize: proto.Uint32(0),
AdminState: proto.Bool(true),
AutoNegMode: proto.Bool(false),
AutoNegMode: proto.Bool(true),
Mtu: proto.Uint32(1514),
},
}, {
Expand Down Expand Up @@ -156,13 +156,13 @@ func TestCreatePort(t *testing.T) {
QosDscpToForwardingClassMap: proto.Uint64(0),
QosMplsExpToForwardingClassMap: proto.Uint64(0),
IpsecPort: proto.Uint64(0),
SupportedSpeed: []uint32{1024},
OperSpeed: proto.Uint32(1024),
SupportedSpeed: []uint32{1000, 10000, 40000},
OperSpeed: proto.Uint32(40000),
SupportedFecMode: []saipb.PortFecMode{saipb.PortFecMode_PORT_FEC_MODE_NONE},
NumberOfIngressPriorityGroups: proto.Uint32(0),
QosMaximumHeadroomSize: proto.Uint32(0),
AdminState: proto.Bool(true),
AutoNegMode: proto.Bool(false),
AutoNegMode: proto.Bool(true),
Mtu: proto.Uint32(1514),
},
}}
Expand Down Expand Up @@ -276,8 +276,9 @@ func TestCreateHostif(t *testing.T) {
Oid: 1,
},
wantAttr: &saipb.HostifAttribute{
Type: saipb.HostifType_HOSTIF_TYPE_NETDEV.Enum(),
ObjId: proto.Uint64(2),
Type: saipb.HostifType_HOSTIF_TYPE_NETDEV.Enum(),
ObjId: proto.Uint64(2),
OperStatus: proto.Bool(true),
},
}}
for _, tt := range tests {
Expand Down
11 changes: 8 additions & 3 deletions dataplane/standalone/saiserver/saiserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ type Server struct {
saipb.UnimplementedEntrypointServer
mgr *attrmgr.AttrMgr
engine *engine.Engine
initialized bool
acl *acl
bfd *bfd
buffer *buffer
Expand Down Expand Up @@ -225,10 +226,14 @@ func (s *Server) ObjectTypeQuery(_ context.Context, req *saipb.ObjectTypeQueryRe
}

func (s *Server) Initialize(ctx context.Context, _ *saipb.InitializeRequest) (*saipb.InitializeResponse, error) {
s.mgr.Reset()
if err := s.engine.Reset(ctx); err != nil {
return nil, err
if s.initialized {
s.mgr.Reset()
s.saiSwitch.Reset()
if err := s.engine.Reset(ctx); err != nil {
return nil, err
}
}
s.initialized = true

return &saipb.InitializeResponse{}, nil
}
Expand Down
4 changes: 4 additions & 0 deletions dataplane/standalone/saiserver/switch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,3 +274,7 @@ func (sw *saiSwitch) PortStateChangeNotification(_ *saipb.PortStateChangeNotific
}
}
}

func (sw saiSwitch) Reset() {
sw.port.Reset()
}
Loading

0 comments on commit 52190cc

Please sign in to comment.