From ec0467efb0e1a9e7e22f73fa2a61905138e5aba7 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Mon, 3 Nov 2014 16:09:57 +1300 Subject: [PATCH 01/52] * Fix ICMP6. * don't busy wait so much on routing updates. * lxc images should be able to use nameservers. --- projectw.sh | 2 +- rfclient/FlowTable.cc | 54 ++++++++++++++++++++++++------------------- rfclient/RFClient.cc | 27 +++++++++++++--------- rfclient/RFClient.hh | 3 ++- rfserver/rfserver.py | 2 +- rftest/create | 4 +++- 6 files changed, 53 insertions(+), 39 deletions(-) diff --git a/projectw.sh b/projectw.sh index 966fc1f1..f71b2c43 100755 --- a/projectw.sh +++ b/projectw.sh @@ -11,7 +11,7 @@ DPPORTNETV6=fc00:: DPPORTS=2 SWITCH1DPID=0x99 MULTITABLEDPS="''" -SATELLITEDPS=$SWITCH1DPID +SATELLITEDPS="''" HOME=/home/projectw RF_HOME=$HOME/RouteFlow diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 8ee3f5ce..b270d7ed 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -168,31 +168,37 @@ void FlowTable::GWResolverCb(FlowTable *ft) { break; } } - - set resolvedRoutes; - set::iterator it; - for (it = ft->unresolvedRoutes.begin(); it != ft->unresolvedRoutes.end(); ++it) { - const RouteEntry& re = ft->routeTable[*it]; - const string addr_str = re.address.toString(); - const string mask_str = re.netmask.toString(); - const string gw_str = re.gateway.toString(); - if (ft->findHost(re.gateway) == MAC_ADDR_NONE) { - syslog(LOG_DEBUG, - "Still cannot resolve gateway %s, will retry route %s/%s", - gw_str.c_str(), addr_str.c_str(), mask_str.c_str()); - ft->resolveGateway(re.gateway, re.interface); - } else { - syslog(LOG_DEBUG, - "Adding previously unresolved route %s/%s via %s", - addr_str.c_str(), mask_str.c_str(), gw_str.c_str()); - ft->sendToHw(RMT_ADD, re); - resolvedRoutes.insert(*it); + if (ft->unresolvedRoutes.size() > 0) { + set resolvedRoutes; + set::iterator it; + uint64_t gatewaysResolved = 0; + for (it = ft->unresolvedRoutes.begin(); it != ft->unresolvedRoutes.end(); ++it) { + const RouteEntry& re = ft->routeTable[*it]; + const string addr_str = re.address.toString(); + const string mask_str = re.netmask.toString(); + const string gw_str = re.gateway.toString(); + if (ft->findHost(re.gateway) == MAC_ADDR_NONE) { + syslog(LOG_DEBUG, + "Still cannot resolve gateway %s, will retry route %s/%s", + gw_str.c_str(), addr_str.c_str(), mask_str.c_str()); + ft->resolveGateway(re.gateway, re.interface); + } else { + syslog(LOG_DEBUG, + "Adding previously unresolved route %s/%s via %s", + addr_str.c_str(), mask_str.c_str(), gw_str.c_str()); + ft->sendToHw(RMT_ADD, re); + resolvedRoutes.insert(*it); + ++gatewaysResolved; + } } - } - for (it = resolvedRoutes.begin(); it != resolvedRoutes.end(); ++it) { - ft->unresolvedRoutes.erase(*it); - } - usleep(100); + if (gatewaysResolved) { + for (it = resolvedRoutes.begin(); it != resolvedRoutes.end(); ++it) { + ft->unresolvedRoutes.erase(*it); + } + } + } else { + usleep(1000); + } } } diff --git a/rfclient/RFClient.cc b/rfclient/RFClient.cc index 16f9fb9f..9a75bbc0 100644 --- a/rfclient/RFClient.cc +++ b/rfclient/RFClient.cc @@ -176,17 +176,20 @@ void RFClient::sendRm(RouteMod rm) { this->rm_q.push(rm); } -RouteMod RFClient::controllerRouteMod(uint32_t port, uint32_t vlan, MACAddress hwaddress, +RouteMod RFClient::controllerRouteMod(uint32_t port, uint32_t vlan, + bool matchMac, MACAddress hwaddress, bool matchIP, const IPAddress &ip_address) { RouteMod rm; rm.set_mod(RMT_CONTROLLER); rm.set_id(this->flowTable->get_vm_id()); rm.set_vm_port(port); - rm.add_match(Match(RFMT_ETHERNET, hwaddress.toString())); + if (matchMac) { + rm.add_match(Match(RFMT_ETHERNET, hwaddress.toString())); + } if (vlan) { rm.add_match(Match(RFMT_VLAN_ID, vlan)); } - if (matchIP ){ + if (matchIP){ if (ip_address.getVersion() == IPV4) { rm.add_match(Match(RFMT_IPV4, ip_address, IPAddress(IPV4, FULL_IPV4_PREFIX))); } else { @@ -208,31 +211,33 @@ void RFClient::sendInterfaceToControllerRouteMods(const Interface &iface) { ++it) { if (it->getVersion() == IPV4) { /* ICMP traffic. */ - rm = controllerRouteMod(port, vlan, hwaddress, true, *it); + rm = controllerRouteMod(port, vlan, true, hwaddress, true, *it); rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_ICMP)); sendRm(rm); /* ARP */ - rm = controllerRouteMod(port, vlan, hwaddress, false, *it); + rm = controllerRouteMod(port, vlan, true, hwaddress, false, *it); rm.add_match(Match(RFMT_ETHERTYPE, (uint16_t)ETHERTYPE_ARP)); sendRm(rm); } else { /* TODO: handle neighbor solicitation et al specifically, - like we do for IPv4 and ARP. */ - rm = controllerRouteMod(port, vlan, hwaddress, true, *it); + like we do for IPv4 and ARP. Will need to implement + IPV6/ICMP6 type code checking. */ + rm = controllerRouteMod(port, vlan, false, hwaddress, false, *it); + rm.add_match(Match(RFMT_ETHERTYPE, (uint16_t)ETHERTYPE_IPV6)); rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_ICMPV6)); + rm.add_option(Option(RFOT_PRIORITY, (uint16_t)(PRIORITY_LOW + 1))); sendRm(rm); - rm = controllerRouteMod(port, vlan, hwaddress, false, *it); + rm = controllerRouteMod(port, vlan, true, hwaddress, true , *it); rm.add_match(Match(RFMT_ETHERTYPE, (uint16_t)ETHERTYPE_IPV6)); rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_ICMPV6)); - rm.add_option(Option(RFOT_PRIORITY, (uint16_t)(PRIORITY_LOW + 1))); sendRm(rm); } /* BGP */ - rm = controllerRouteMod(port, vlan, hwaddress, true, *it); + rm = controllerRouteMod(port, vlan, true, hwaddress, true, *it); rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_TCP)); rm.add_match(Match(RFMT_TP_SRC, (uint16_t)TPORT_BGP)); sendRm(rm); - rm = controllerRouteMod(port, vlan, hwaddress, true, *it); + rm = controllerRouteMod(port, vlan, true, hwaddress, true, *it); rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_TCP)); rm.add_match(Match(RFMT_TP_DST, (uint16_t)TPORT_BGP)); sendRm(rm); diff --git a/rfclient/RFClient.hh b/rfclient/RFClient.hh index 60ff8d19..a21c8c1f 100644 --- a/rfclient/RFClient.hh +++ b/rfclient/RFClient.hh @@ -39,7 +39,8 @@ class RFClient : private RFProtocolFactory, private IPCMessageProcessor, bool process(const string &from, const string &to, const string &channel, IPCMessage& msg); void sendRm(RouteMod rm); - RouteMod controllerRouteMod(uint32_t port, uint32_t vlan, MACAddress hwaddress, + RouteMod controllerRouteMod(uint32_t port, uint32_t vlan, + bool matchMac, MACAddress hwaddress, bool matchIP, const IPAddress &ip_address); void sendInterfaceToControllerRouteMods(const Interface &iface); void sendAllInterfaceToControllerRouteMods(uint32_t vm_port); diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 580a6f06..fab0e5b0 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -88,7 +88,7 @@ def configure_datapath(self): # default drop rm = RouteMod(RMT_ADD, self.dp_id) - #rm.add_match(Match.ETHERTYPE(ETHERTYPE_IP)) + rm.add_match(Match.ETHERTYPE(ETHERTYPE_IP)) rm.add_option(self.DROP_PRIORITY) rms.append(rm) diff --git a/rftest/create b/rftest/create index 44ed9641..4dd3b034 100755 --- a/rftest/create +++ b/rftest/create @@ -13,13 +13,15 @@ apt-get -y --force-yes install lxc mkdir -p $LXCDIR lxc-create -t ubuntu -n base || exit 1 +cp -f /etc/resolv.conf $LXCDIR/base/rootfs/etc + chroot $LXCDIR/base/rootfs apt-get update # !!! # ADD THE PACKETS YOU NEED TO INSTALL HERE # !!! chroot $LXCDIR/base/rootfs apt-get -y --force-yes install quagga \ libboost-thread-dev libboost-system-dev libboost-filesystem-dev \ - libboost-program-options-dev rsyslog vlan tcpdump + libboost-program-options-dev rsyslog vlan tcpdump telnet # !!! if [ -e /usr/local/lib/libzmq.so ]; then From e10bc1159c8f5545f9b63683d67ea6bfc3b60ea1 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Thu, 6 Nov 2014 21:31:20 +1300 Subject: [PATCH 02/52] Pica8 workaround no longer necessary as of 2.4.1S1. --- rfserver/rfserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index fab0e5b0..580a6f06 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -88,7 +88,7 @@ def configure_datapath(self): # default drop rm = RouteMod(RMT_ADD, self.dp_id) - rm.add_match(Match.ETHERTYPE(ETHERTYPE_IP)) + #rm.add_match(Match.ETHERTYPE(ETHERTYPE_IP)) rm.add_option(self.DROP_PRIORITY) rms.append(rm) From 8ff0524e5508d28f03b54df9b9aeeedda667eaeb Mon Sep 17 00:00:00 2001 From: Vikram Dham Date: Thu, 6 Nov 2014 16:05:39 -0800 Subject: [PATCH 03/52] Changes to have rftest2 work in vandervecken branch - Modified rfclient/RFClient.cc - Modified rftest/rftest2 --- rfclient/RFClient.cc | 10 ++++++++++ rftest/rftest2 | 30 +++++++++++++++++------------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/rfclient/RFClient.cc b/rfclient/RFClient.cc index 9a75bbc0..dc9451b3 100644 --- a/rfclient/RFClient.cc +++ b/rfclient/RFClient.cc @@ -241,6 +241,16 @@ void RFClient::sendInterfaceToControllerRouteMods(const Interface &iface) { rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_TCP)); rm.add_match(Match(RFMT_TP_DST, (uint16_t)TPORT_BGP)); sendRm(rm); + /* OSPF for IPv4 */ + rm = controllerRouteMod(port, vlan, false, hwaddress, false, *it); + rm.add_match(Match(RFMT_ETHERTYPE, (uint16_t)ETHERTYPE_IP)); + rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_OSPF)); + sendRm(rm); + /* OSPF for IPv6 */ + rm = controllerRouteMod(port, vlan, false, hwaddress, false, *it); + rm.add_match(Match(RFMT_ETHERTYPE, (uint16_t)ETHERTYPE_IPV6)); + rm.add_match(Match(RFMT_NW_PROTO, (uint16_t)IPPROTO_OSPF)); + sendRm(rm); /* TODO: add other IGP traffic here - RIPv2 et al */ } } diff --git a/rftest/rftest2 b/rftest/rftest2 index 845cc877..f4be9bfc 100755 --- a/rftest/rftest2 +++ b/rftest/rftest2 @@ -7,6 +7,7 @@ fi SCRIPT_NAME="rftest2" LXCDIR=/var/lib/lxc +IPC="zmq" MONGODB_CONF=/etc/mongodb.conf MONGODB_PORT=27017 MONGODB_ADDR=192.168.10.1 @@ -16,6 +17,7 @@ if [ -d ./build ]; then else RF_HOME=.. fi + export PATH=$PATH:/usr/local/bin:/usr/local/sbin export PYTHONPATH=$PYTHONPATH:$RF_HOME @@ -95,20 +97,25 @@ trap "reset 0; exit 0" INT echo_bold "-> Setting up the management bridge (lxcbr0)..." ifconfig lxcbr0 $MONGODB_ADDR up -echo_bold "-> Setting up MongoDB..." -sed -i "/bind_ip/c\bind_ip = 127.0.0.1,$MONGODB_ADDR" $MONGODB_CONF -service mongodb restart -wait_port_listen $MONGODB_PORT +if [ $IPC = "mongodb" ]; then + echo_bold "-> Setting up MongoDB..." + sed -i "/bind_ip/c\bind_ip = 127.0.0.1,$MONGODB_ADDR" $MONGODB_CONF + service mongodb restart + wait_port_listen $MONGODB_PORT +fi + +echo_bold "-> Starting RFServer..." +nice -n 20 ./rfserver/rfserver.py rftest/rftest2config.csv & echo_bold "-> Configuring the virtual machines..." for vm in rfvmA rfvmB rfvmC rfvmD; do mkdir "/var/lib/lxc/$vm/rootfs/opt/rfclient" - cp build/rfclient "/var/lib/lxc/$vm/rootfs/opt/rfclient/rfclient" + cp rfclient/rfclient "/var/lib/lxc/$vm/rootfs/opt/rfclient/rfclient" # We sleep for a few seconds to wait for the interfaces to go up - echo < "/var/lib/lxc/$vm/rootfs/root/run_rfclient.sh" -#!/bin/sh" + cat < "/var/lib/lxc/$vm/rootfs/root/run_rfclient.sh" +#!/bin/sh sleep 3 /etc/init.d/quagga start /opt/rfclient/rfclient 2>&1 >/var/log/rfclient.log @@ -120,14 +127,9 @@ EOF done echo_bold "-> Starting the controller and RFPRoxy..." -cd ryu-rfproxy -ryu-manager rfproxy.py & -cd - +ryu-manager --use-stderr --ofp-tcp-listen-port=$CONTROLLER_PORT ryu-rfproxy/rfproxy.py & wait_port_listen $CONTROLLER_PORT -echo_bold "-> Starting RFServer..." -./rfserver/rfserver.py rftest/rftest2config.csv & - echo_bold "-> Starting the control plane network (dp0 VS)..." ovs-vsctl add-br dp0 for i in 1 2 3; do @@ -139,6 +141,8 @@ ovs-vsctl add-port dp0 rfvmA.4 ovs-vsctl add-port dp0 rfvmD.4 ovs-vsctl set Bridge dp0 other-config:datapath-id=7266767372667673 ovs-vsctl set-controller dp0 tcp:127.0.0.1:$CONTROLLER_PORT +ovs-vsctl set Bridge dp0 protocols=OpenFlow13 +ovs-ofctl -O OpenFlow13 add-flow dp0 actions=CONTROLLER:65509 echo_bold "---" echo_bold "This test is up and running." From 412ec4b72412e4c86c3588cc9a9ff98d73a030a1 Mon Sep 17 00:00:00 2001 From: Stacey Sheldon Date: Sat, 8 Nov 2014 12:41:16 -0500 Subject: [PATCH 04/52] rfclient: apply setsockopt(SO_RCVBUFFORCE) to netlink socket for routes The receive buffer was being forced twice on the neighbour update socket. The second call should have been asserting the receive buffer increase on the route update socket. Without this, large routing update bursts were resulting in dropped netlink messages, and thus missing FIB updates in the OF dataplane. These dropped netlink messages can be seen by running: $ cat /proc/net/netlink and looking for non-zero values in the "Drops" column. Signed-off-by: Stacey Sheldon --- rfclient/FlowTable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index b270d7ed..6dc9f37d 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -66,7 +66,7 @@ void FlowTable::operator()() { syslog(LOG_NOTICE, "Netlink interface enabled"); rtnl_open(&rth, RTMGRP_IPV4_ROUTE | RTMGRP_IPV6_ROUTE | RTMGRP_IPV4_MROUTE | RTMGRP_IPV6_MROUTE); - int rs = setsockopt(rthNeigh.fd, SOL_SOCKET, SO_RCVBUFFORCE, &nl_buffersize, sizeof(nl_buffersize)); + int rs = setsockopt(rth.fd, SOL_SOCKET, SO_RCVBUFFORCE, &nl_buffersize, sizeof(nl_buffersize)); if (rs != 0) { syslog(LOG_CRIT, "cannot set socket size for routes: %d", errno); exit(rs); From 9070a9028b65238160d53df199659ff2866685a3 Mon Sep 17 00:00:00 2001 From: Stacey Sheldon Date: Sat, 8 Nov 2014 12:34:13 -0500 Subject: [PATCH 05/52] projectw.sh: fix path to external config files for rfserver The paths for the external (non-default) config files was missing a '/' which meant that the startup script would never find the external config files and would always use the default files in /tmp. Signed-off-by: Stacey Sheldon --- projectw.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projectw.sh b/projectw.sh index f71b2c43..4a2365a7 100755 --- a/projectw.sh +++ b/projectw.sh @@ -17,8 +17,8 @@ HOME=/home/projectw RF_HOME=$HOME/RouteFlow RFSERVERCONFIG=/tmp/rfserverconfig.csv RFSERVERINTERNAL=/tmp/rfserverinternal.csv -HOME_RFSERVERCONFIG="$HOME"`basename $RFSERVERCONFIG` -HOME_RFSERVERINTERNAL="$HOME"`basename $RFSERVERINTERNAL` +HOME_RFSERVERCONFIG="$HOME/"`basename $RFSERVERCONFIG` +HOME_RFSERVERINTERNAL="$HOME/"`basename $RFSERVERINTERNAL` CONTROLLER_PORT=6653 LXCDIR=/var/lib/lxc RFVM1=$LXCDIR/rfvm1 From b17fec14b948b13d4349e1212b0c14dadb3a89dd Mon Sep 17 00:00:00 2001 From: Vikram Dham Date: Tue, 11 Nov 2014 10:01:39 -0800 Subject: [PATCH 06/52] Added the OSPF constant defines for enabling rftest2 in vandervecken branch --- rflib/defs.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rflib/defs.h b/rflib/defs.h index 7657c5cf..91406d77 100644 --- a/rflib/defs.h +++ b/rflib/defs.h @@ -61,6 +61,8 @@ typedef enum port_config_type { #define TPORT_BGP 0x00B3 #define OFPP_CONTROLLER 0xFFFFFFFD +#define IPPROTO_OSPF 0x59 + #define CONTROLLER_GROUP 1 #endif /* __DEFS_H__ */ From c4519395b3648404b8245fe02040e09f89087152 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Tue, 25 Nov 2014 16:59:09 +1300 Subject: [PATCH 07/52] NoviFlow now works well with different priorities in same table. --- rfserver/rfserver.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 580a6f06..c0a9d610 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -159,9 +159,6 @@ def handle_isl_route_mod(self, r, rm): class NoviFlowMultitableRouteModTranslator(RouteModTranslator): - # NoviFlow as of Aug 2014 doesn't handle flow add operations - # in unsorted priority order well (very slow). For the moment - # make all flows in a table same priority. FIB_TABLE = 2 ETHER_TABLE = 1 @@ -262,18 +259,12 @@ def handle_route_mod(self, entry, rm): rm.add_action(Action.OUTPUT(entry.dp_port)) rm.set_table(self.FIB_TABLE) - # See NoviFlow note - rm.set_options(None) - rm.add_option(Option.PRIORITY(PRIORITY_HIGH)) rms.extend(self._send_rm_with_matches(rm, entry.dp_port, entries)) return rms def handle_isl_route_mod(self, r, rm): rms = [] - # See NoviFlow note - rm.set_options(None) - rm.add_option(Option.PRIORITY(PRIORITY_HIGH)) rm.set_id(self.dp_id) rm.set_table(self.FIB_TABLE) rm.set_actions(None) From 27699b335049a8afbb9fad828b3d6521142dd107 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Nov 2014 15:52:46 +1300 Subject: [PATCH 08/52] Send RouteMods via a Queue to rfproxy. First part of adding flow control support (towards rfproxy). --- rfserver/rfserver.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index c0a9d610..ad2cc2d6 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -9,6 +9,7 @@ import time import copy import Queue +import threading from bson.binary import Binary @@ -467,6 +468,7 @@ def handle_route_mod(self, entry, rm): return rms class RFServer(RFProtocolFactory, IPC.IPCMessageProcessor): + def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.config = RFConfig(configfile) self.islconf = RFISLConf(islconffile) @@ -495,10 +497,28 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.log.info("Datapaths that support multiple tables: %s", list(self.multitabledps)) + self.dp_q = Queue.Queue() + self.ipc_lock = threading.Lock() self.ipc = IPCService.for_server(RFSERVER_ID) + + self.worker = threading.Thread(target=self.dp_worker) + self.worker.daemon = True + self.worker.start() + self.ipc.listen(RFCLIENT_RFSERVER_CHANNEL, self, self, False) self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, True) + def ipc_send(self, channel, channel_id, msg): + self.ipc_lock.acquire() + self.ipc.send(channel, channel_id, msg) + self.ipc_lock.release() + + def dp_worker(self): + while True: + (ct_id, rm) = self.dp_q.get(block=True) + self.ipc_send(RFSERVER_RFPROXY_CHANNEL, ct_id, rm) + self.dp_q.task_done() + def process(self, from_, to, channel, msg): type_ = msg.get_type() if type_ == PORT_REGISTER: @@ -553,13 +573,13 @@ def register_vm_port(self, vm_id, vm_port, eth_addr): format_id(entry.dp_id), entry.dp_port)) def acknowledge_route_mod(self, ct_id, vm_id, vm_port): - self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), + self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK)) return def send_route_mod(self, ct_id, rm): rm.add_option(Option.CT_ID(ct_id)) - self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(ct_id), rm) + self.dp_q.put((str(ct_id), rm)) # Handle RouteMod messages (type ROUTE_MOD) # @@ -756,7 +776,7 @@ def set_dp_port_down(self, ct_id, dp_id, dp_port): def reset_vm_port(self, vm_id, vm_port): if vm_id is None: return - self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), + self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_RESET)) self.log.info("Resetting client port (vm_id=%s, vm_port=%i)" % @@ -772,10 +792,10 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): msg = DataPlaneMap(ct_id=entry.ct_id, dp_id=entry.dp_id, dp_port=entry.dp_port, vs_id=vs_id, vs_port=vs_port) - self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + self.ipc_send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) msg = PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_MAP_SUCCESS) - self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) + self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) self.log.info("Mapping client-datapath association " "(vm_id=%s, vm_port=%i, dp_id=%s, " "dp_port=%i, vs_id=%s, vs_port=%i)" % From 56781436b791910b65620bd2cabdff3a33493a59 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Nov 2014 16:20:24 +1300 Subject: [PATCH 09/52] Set outstanding RouteMod limit to 1 (from 10). --- rfclient/RFClient.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfclient/RFClient.hh b/rfclient/RFClient.hh index a21c8c1f..6fcee119 100644 --- a/rfclient/RFClient.hh +++ b/rfclient/RFClient.hh @@ -11,7 +11,7 @@ #include "FlowTable.hh" #include "PortMapper.hh" -const int max_rm_outstanding = 10; +const int max_rm_outstanding = 1; class RFClient : private RFProtocolFactory, private IPCMessageProcessor, public InterfaceMap { From 2482b67dec49860cc07d7eb562fc866397492790 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Nov 2014 21:04:37 +1300 Subject: [PATCH 10/52] End to end flow control (also needs patch to rfproxy to send back RouteMod when it has been sent to datapath). Don't acknowledge a RouteMod from rfclient, until rfproxy acknowledges it. Probably too aggressive, but more important that is correct at this stage (ie. RouteMods must not be lost). --- rfserver/rfserver.py | 52 +++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index ad2cc2d6..18a97c41 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -497,8 +497,10 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.log.info("Datapaths that support multiple tables: %s", list(self.multitabledps)) + self.ack_q = Queue.Queue() self.dp_q = Queue.Queue() self.ipc_lock = threading.Lock() + self.routemod_outstanding = threading.Event() self.ipc = IPCService.for_server(RFSERVER_ID) self.worker = threading.Thread(target=self.dp_worker) @@ -512,29 +514,39 @@ def ipc_send(self, channel, channel_id, msg): self.ipc_lock.acquire() self.ipc.send(channel, channel_id, msg) self.ipc_lock.release() - + def dp_worker(self): while True: (ct_id, rm) = self.dp_q.get(block=True) self.ipc_send(RFSERVER_RFPROXY_CHANNEL, ct_id, rm) self.dp_q.task_done() + def send_routemod_acks(self): + while not self.ack_q.empty(): + (vm_id, ack) = self.ack_q.get() + self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, vm_id, ack) + self.ack_q.task_done() + def process(self, from_, to, channel, msg): type_ = msg.get_type() - if type_ == PORT_REGISTER: - self.register_vm_port(msg.get_vm_id(), msg.get_vm_port(), - msg.get_hwaddress()) - elif type_ == ROUTE_MOD: - self.register_route_mod(msg) - elif type_ == DATAPATH_PORT_REGISTER: - self.register_dp_port(msg.get_ct_id(), - msg.get_dp_id(), - msg.get_dp_port()) - elif type_ == DATAPATH_DOWN: - self.set_dp_down(msg.get_ct_id(), msg.get_dp_id()) - elif type_ == VIRTUAL_PLANE_MAP: - self.map_port(msg.get_vm_id(), msg.get_vm_port(), - msg.get_vs_id(), msg.get_vs_port()) + if channel == RFCLIENT_RFSERVER_CHANNEL: + if type_ == ROUTE_MOD: + self.register_route_mod(msg) + elif type_ == PORT_REGISTER: + self.register_vm_port(msg.get_vm_id(), msg.get_vm_port(), + msg.get_hwaddress()) + elif channel == RFSERVER_RFPROXY_CHANNEL: + if type_ == DATAPATH_PORT_REGISTER: + self.register_dp_port(msg.get_ct_id(), + msg.get_dp_id(), + msg.get_dp_port()) + elif type_ == DATAPATH_DOWN: + self.set_dp_down(msg.get_ct_id(), msg.get_dp_id()) + elif type_ == VIRTUAL_PLANE_MAP: + self.map_port(msg.get_vm_id(), msg.get_vm_port(), + msg.get_vs_id(), msg.get_vs_port()) + elif type_ == ROUTE_MOD: + self.send_routemod_acks() # Port register methods def register_vm_port(self, vm_id, vm_port, eth_addr): @@ -572,10 +584,9 @@ def register_vm_port(self, vm_id, vm_port, eth_addr): % (format_id(vm_id), vm_port, eth_addr, format_id(entry.dp_id), entry.dp_port)) - def acknowledge_route_mod(self, ct_id, vm_id, vm_port): - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), - PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK)) - return + def queue_routemod_ack(self, ct_id, vm_id, vm_port): + self.ack_q.put((str(vm_id), + PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK))) def send_route_mod(self, ct_id, rm): rm.add_option(Option.CT_ID(ct_id)) @@ -625,7 +636,8 @@ def register_route_mod(self, rm): for rm in rms: self.send_route_mod(entry.ct_id, rm) - self.acknowledge_route_mod(entry.ct_id, vm_id, vm_port) + + self.queue_routemod_ack(entry.ct_id, vm_id, vm_port) # DatapathPortRegister methods def register_dp_port(self, ct_id, dp_id, dp_port): From 72cb580108d65461dd0a796736d419855c991207 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Nov 2014 21:08:52 +1300 Subject: [PATCH 11/52] Need NoviFlow priority workaround still (with differing priorities in same table, FlowMods still slow) --- rfserver/rfserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 18a97c41..6c0d6eb8 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -261,6 +261,9 @@ def handle_route_mod(self, entry, rm): rm.set_table(self.FIB_TABLE) + rm.set_options(None) + rm.add_option(self.CONTROLLER_PRIORITY) + rms.extend(self._send_rm_with_matches(rm, entry.dp_port, entries)) return rms @@ -268,6 +271,8 @@ def handle_isl_route_mod(self, r, rm): rms = [] rm.set_id(self.dp_id) rm.set_table(self.FIB_TABLE) + rm.set_options(None) + rm.add_option(self.CONTROLLER_PRIORITY) rm.set_actions(None) rm.add_action(Action.SET_ETH_SRC(r.eth_addr)) rm.add_action(Action.SET_ETH_DST(r.rem_eth_addr)) From e901a53918b1181a669761e671f8d5cb398e9007 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Nov 2014 21:11:25 +1300 Subject: [PATCH 12/52] Correct Corsa deferred VLAN swap operation. --- rfserver/rfserver.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 6c0d6eb8..76cc4f33 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -389,12 +389,17 @@ def _send_rm_with_matches(self, rm, out_port, entries): if (entry.get_status() == RFENTRY_ACTIVE or entry.get_status() == RFISL_ACTIVE): dst_eth = None - for action_dict in rm.actions: + actions = rm.actions + rm.set_actions(None) + for action_dict in actions: action = Action.from_dict(action_dict) action_type = action.type_to_str(action._type) if action_type == 'RFAT_SET_ETH_DST': dst_eth = action.get_value() - break + elif action_type == 'RFAT_SWAP_VLAN_ID': + vlan_id = action.get_value() + action = Action.SET_VLAN_ID(vlan_id) + rm.add_action(action) if dst_eth not in self.actions_to_groupid: self.last_groupid += 1 new_groupid = self.last_groupid From 317d227e42c12b12aed5767b5662f6f8fbc2856a Mon Sep 17 00:00:00 2001 From: Vincenzo Maffione Date: Thu, 11 Dec 2014 09:18:47 +0100 Subject: [PATCH 13/52] rflib: TLV: fix shared_array constructor invokation This avoids a compilation problem. Signed-off-by: Chris --- rflib/types/TLV.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rflib/types/TLV.cc b/rflib/types/TLV.cc index f3cda824..98549f28 100644 --- a/rflib/types/TLV.cc +++ b/rflib/types/TLV.cc @@ -204,7 +204,7 @@ uint8_t TLV::type_from_BSON(mongo::BSONObj bson) { boost::shared_array TLV::value_from_BSON(mongo::BSONObj bson, byte_order order) { - boost::shared_array arr(NULL); + boost::shared_array arr; const mongo::BSONElement& bvalue = bson["value"]; if (bvalue.type() != mongo::BinData) From 0a8b21a0b7aeae52e1ba7ac14e66e4de2ddb41a4 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 7 Jan 2015 13:30:00 +1300 Subject: [PATCH 14/52] Support dynamic mapping Initial push as for backup Details of change will be explained later Signed-off-by: Trung Truong --- rflib/defs.py | 13 +- rfserver/rfserver.py | 346 +++++++++++++++++++++++++++++++++---------- rfserver/rftable.py | 118 ++++++++++++++- 3 files changed, 395 insertions(+), 82 deletions(-) diff --git a/rflib/defs.py b/rflib/defs.py index d41d7bcb..a58eb845 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -2,7 +2,7 @@ IPC_TYPE = 'zeromq' # options are 'zeromq' or 'mongo' DB_TYPE = 'memory' # options are 'memory' or 'mongo' -MONGO_ADDRESS = "192.168.10.1:27017" +MONGO_ADDRESS = "127.0.0.1:27017" MONGO_DB_NAME = "db" ZEROMQ_ADDRESS = "tcp://192.168.10.1:25555" @@ -15,6 +15,9 @@ RFISL_NAME = "rfisl" RFISLCONF_NAME = "rfislconf" +RFVMPORTTABLE_NAME = "vmporttable" +RFDPPORTTABLE_NAME = "dpporttable" + RFSERVER_ID = "rfserver" RFPROXY_ID = "rfproxy" @@ -64,8 +67,14 @@ PC_MAP = 0 PC_RESET = 1 +# Data plane Port Map Configuration (DCT_) +DCT_UPDATE_DP = 0 # Update Proxy map table by DP info +DCT_UPDATE_VS = 1 # Update Proxy map table by VS info +DCT_DELETE_DP = 2 # Delete Proxy map table by DP info +DCT_DELETE_VS = 3 # Delete Proxy map table by VS info + # Format 12-digit hex ID -format_id = lambda dp_id: hex(dp_id).rstrip("L") +format_id = lambda dp_id: hex(dp_id).rstrip("L") if (dp_id is not None) else 'None' netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) cidr_to_mask = lambda a: ((1 << a) - 1) << (32 - a) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 76cc4f33..fd1fdece 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -9,7 +9,7 @@ import time import copy import Queue -import threading +import signal from bson.binary import Binary @@ -24,6 +24,8 @@ from rftable import * +from rfserverrpc import RFServerRPC + logging.basicConfig( level=logging.INFO, format='%(asctime)s %(name)-15s %(levelname)-8s %(message)s', @@ -35,7 +37,6 @@ REGISTER_ASSOCIATED = 1 REGISTER_ISL = 2 - class RouteModTranslator(object): DROP_PRIORITY = Option.PRIORITY(PRIORITY_LOWEST + PRIORITY_BAND) @@ -48,7 +49,7 @@ def __init__(self, dp_id, ct_id, rftable, isltable, log): self.rftable = rftable self.isltable = isltable self.log = log - + def configure_datapath(self): raise Exception @@ -108,6 +109,7 @@ def handle_controller_route_mod(self, entry, rm): def handle_route_mod(self, entry, rm): rms = [] + entries = self.rftable.get_entries(dp_id=entry.dp_id, ct_id=entry.ct_id) entries.extend(self.isltable.get_entries(dp_id=entry.dp_id, @@ -130,7 +132,17 @@ def handle_isl_route_mod(self, r, rm): entries = self.rftable.get_entries(dp_id=r.dp_id, ct_id=r.ct_id) rms.extend(self._send_rm_with_matches(rm, r.dp_port, entries)) return rms - + # This method used to remove flow entries from switches when + # users make a change (i.e. delete) of an association + def dp_flows_remove_by_vm_port(self, entry): + rms = [] + rm = RouteMod(RMT_DELETE, self.dp_id) + rm.add_match(Match.IN_PORT(entry.dp_port)) + rm.add_match(Match.ETHERNET(entry.eth_addr)) + rm.add_option(self.DEFAULT_PRIORITY) + rms.append(rm) + print rm + return rms class SatelliteRouteModTranslator(DefaultRouteModTranslator): @@ -158,8 +170,10 @@ def handle_isl_route_mod(self, r, rm): rms.extend(self._send_rm_with_matches(rm, r.dp_port, entries)) return rms - class NoviFlowMultitableRouteModTranslator(RouteModTranslator): + # NoviFlow as of Aug 2014 doesn't handle flow add operations + # in unsorted priority order well (very slow). For the moment + # make all flows in a table same priority. FIB_TABLE = 2 ETHER_TABLE = 1 @@ -260,19 +274,20 @@ def handle_route_mod(self, entry, rm): rm.add_action(Action.OUTPUT(entry.dp_port)) rm.set_table(self.FIB_TABLE) - + # See NoviFlow note rm.set_options(None) - rm.add_option(self.CONTROLLER_PRIORITY) + rm.add_option(Option.PRIORITY(PRIORITY_HIGH)) rms.extend(self._send_rm_with_matches(rm, entry.dp_port, entries)) return rms def handle_isl_route_mod(self, r, rm): rms = [] + # See NoviFlow note + rm.set_options(None) + rm.add_option(Option.PRIORITY(PRIORITY_HIGH)) rm.set_id(self.dp_id) rm.set_table(self.FIB_TABLE) - rm.set_options(None) - rm.add_option(self.CONTROLLER_PRIORITY) rm.set_actions(None) rm.add_action(Action.SET_ETH_SRC(r.eth_addr)) rm.add_action(Action.SET_ETH_DST(r.rem_eth_addr)) @@ -389,17 +404,12 @@ def _send_rm_with_matches(self, rm, out_port, entries): if (entry.get_status() == RFENTRY_ACTIVE or entry.get_status() == RFISL_ACTIVE): dst_eth = None - actions = rm.actions - rm.set_actions(None) - for action_dict in actions: + for action_dict in rm.actions: action = Action.from_dict(action_dict) action_type = action.type_to_str(action._type) if action_type == 'RFAT_SET_ETH_DST': dst_eth = action.get_value() - elif action_type == 'RFAT_SWAP_VLAN_ID': - vlan_id = action.get_value() - action = Action.SET_VLAN_ID(vlan_id) - rm.add_action(action) + break if dst_eth not in self.actions_to_groupid: self.last_groupid += 1 new_groupid = self.last_groupid @@ -478,10 +488,10 @@ def handle_route_mod(self, entry, rm): return rms class RFServer(RFProtocolFactory, IPC.IPCMessageProcessor): - def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.config = RFConfig(configfile) self.islconf = RFISLConf(islconffile) + try: self.multitabledps = set([int(x, 16) for x in multitabledps.split(",")]) except ValueError: @@ -494,6 +504,12 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): # Initialise state tables self.rftable = RFTable() self.isltable = RFISLTable() + + # Initalize two new tables for VM ports and DP ports state + # At this stage, the tables keep simple port info as RFTable does + #TODO: 1. Add more port state info (e.g. speed, desc, medium type...). 2. Link RFTable to these tables to avoid duplicate data + self.vmporttable = RFVMPortTable() + self.dpporttable = RFDPPortTable() self.route_mod_translator = {} @@ -507,59 +523,189 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.log.info("Datapaths that support multiple tables: %s", list(self.multitabledps)) - self.ack_q = Queue.Queue() - self.dp_q = Queue.Queue() - self.ipc_lock = threading.Lock() - self.routemod_outstanding = threading.Event() self.ipc = IPCService.for_server(RFSERVER_ID) - - self.worker = threading.Thread(target=self.dp_worker) - self.worker.daemon = True - self.worker.start() - self.ipc.listen(RFCLIENT_RFSERVER_CHANNEL, self, self, False) - self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, True) - - def ipc_send(self, channel, channel_id, msg): - self.ipc_lock.acquire() - self.ipc.send(channel, channel_id, msg) - self.ipc_lock.release() + self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, False) + + # Delete a mapping configuration. + def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + cf_entry = None + if vm_id is not None and vm_port is not None: + cf_entry = self.config.get_config_for_vm_port(vm_id, vm_port) + + elif dp_id is not None and dp_port is not None and ct_id is not None: + cf_entry = self.config.get_config_for_dp_port(ct_id, dp_id, dp_port) + + if cf_entry is not None: #Delete it from config table + self.config.remove_entry(cf_entry) + # Send reset signal to RFClient, so it starts generating PortMap msg + #self.reset_vm_port(cf_entry.vm_id, cf_entry.vm_port) + # Update map table in RFProxy + rf_entry = self.rftable.get_entry_by_vm_port(cf_entry.vm_id, cf_entry.vm_port) + if rf_entry is not None: + # Delete from RFTable & send a reset msg to RFProxy + self.rftable.remove_entry(rf_entry) + if rf_entry.get_status() == RFENTRY_ACTIVE: + msg = DataPlaneMap(ct_id=rf_entry.ct_id, + dp_id=rf_entry.dp_id, dp_port=rf_entry.dp_port, + vs_id=rf_entry.vs_id, vs_port=rf_entry.vs_port, operation_id=DCT_DELETE_DP) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(rf_entry.ct_id), msg) + self.log.info("Resetting RFProxy map table (dp_id=%s, dp_port=%i) - (vs_id=%s, vs_port=%i)" % + (format_id(rf_entry.dp_id), rf_entry.dp_port, format_id(rf_entry.vs_id), rf_entry.vs_port)) + #TODO: Update flow table of related dp + translator = self.route_mod_translator[rf_entry.dp_id] + rms = translator.dp_flows_remove_by_vm_port(rf_entry) + for rm in rms: + self.send_route_mod(rf_entry.ct_id, rm) + return True + + self.log.info("No config found") + return False + + # Update or create a mapping. If not exist, create it. + # A VM port can be mapped to a new DP port if the DP port is not being used, similar to DP port mapping change. + def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + if vm_id is None or vm_port is None \ + or dp_id is None or dp_port is None or ct_id is None: + return False + + vm_port_conf = self.config.get_config_for_vm_port(vm_id=vm_id, vm_port=vm_port) + dp_port_conf = self.config.get_config_for_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + old_conf = None + rf_entry = None + action = None + # 1. Update CONFIG table + if vm_port_conf is not None: + if dp_port_conf is not None: + # Both are in use, so do nothing + self.log.info("Both VM port and DP port are in used. No update can be done") + return False + else: # Map VM port to a new DP port + vm_port_conf.update_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + self.config.set_entry(vm_port_conf) + action = 1 + else: # Map DP port to a new VM port + if dp_port_conf is not None: # Map DP port to a new VM port + dp_port_conf.update_vm_port(vm_id=vm_id, vm_port=vm_port) + self.config.set_entry(dp_port_conf) + action = 2 + else: # Create a new mapping between VM port & DP port + self.config.set_entry(RFConfigEntry(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port)) + + action = 3 + + # 2. Update RFTable + rfclient_inform = False + if action == 1: # If update mapping of VM port + rf_entry = self.rftable.get_entry_by_vm_port(vm_id=vm_id, vm_port=vm_port) + if rf_entry is None: + rf_entry = RFEntry() + dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if dp_port_info is not None: + rf_entry.update_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if rf_entry.get_status() == RFENTRY_ACTIVE: + rfproxy_oper_id = DCT_UPDATE_VS + rfclient_inform = True + else: # We have no info about the new dp port + rf_entry.make_idle(RFENTRY_IDLE_VM_PORT) + # Reset rfproxy table + rfproxy_oper_id = DCT_DELETE_VS + # Update to RFTable + self.rftable.set_entry(rf_entry) + elif action == 2: # Update mapping of DP port + rf_entry = self.rftable.get_entry_by_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if rf_entry is None: + rf_entry = RFEntry() + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) + if vm_port_info is not None: + rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, + vs_id=vm_port_info.vs_id, + vs_port=vm_port_info.vs_port, + eth_addr=vm_port_info.eth_addr) + self.rftable.set_entry(rf_entry) + if rf_entry.get_status() == RFENTRY_ACTIVE: + rfproxy_oper_id = DCT_UPDATE_DP + rfclient_inform = True + else: # We have no info about the new vm port + rf_entry.make_idle(RFENTRY_IDLE_DP_PORT) + self.rftable.set_entry(rf_entry) + rfproxy_oper_id = DCT_DELETE_DP + else: # Create a new mapping + rf_entry = RFEntry() + dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) + + if dp_port_info is not None: + rf_entry.update_dp_port(dp_id=dp_id, dp_port=dp_port, ct_id=ct_id) + + if vm_port_info is not None: + rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, + vs_id=vm_port_info.vs_id, + vs_port=vm_port_info.vs_port, + eth_addr=vm_port_info.eth_addr) + + if vm_port_info is None and dp_port_info is None: + rfproxy_oper_id = None + else: + self.rftable.set_entry(rf_entry) + if rf_entry.get_status() == RFENTRY_ACTIVE: + rfproxy_oper_id = DCT_UPDATE_DP + rfclient_inform = True + + if rfproxy_oper_id is not None: + msg = DataPlaneMap(ct_id=rf_entry.ct_id, + dp_id=rf_entry.dp_id, + dp_port=rf_entry.dp_port, + vs_id=rf_entry.vs_id, + vs_port=rf_entry.vs_port, + operation_id=rfproxy_oper_id) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(rf_entry.ct_id), msg) + + if rfclient_inform: + # Inform RFClient about the new mapping + msg = PortConfig(vm_id=vm_id, vm_port=vm_port, + operation_id=PCT_MAP_SUCCESS) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(rf_entry.vm_id), msg) + + self.log.info("Updating a new client-datapath association " + "(vm_id=%s, vm_port=%s, dp_id=%s, " + "dp_port=%s, vs_id=%s, vs_port=%s)" % + (format_id(rf_entry.vm_id), rf_entry.vm_port, + format_id(rf_entry.dp_id), rf_entry.dp_port, + format_id(rf_entry.vs_id), rf_entry.vs_port)) + return True - def dp_worker(self): - while True: - (ct_id, rm) = self.dp_q.get(block=True) - self.ipc_send(RFSERVER_RFPROXY_CHANNEL, ct_id, rm) - self.dp_q.task_done() - - def send_routemod_acks(self): - while not self.ack_q.empty(): - (vm_id, ack) = self.ack_q.get() - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, vm_id, ack) - self.ack_q.task_done() - def process(self, from_, to, channel, msg): type_ = msg.get_type() - if channel == RFCLIENT_RFSERVER_CHANNEL: - if type_ == ROUTE_MOD: - self.register_route_mod(msg) - elif type_ == PORT_REGISTER: - self.register_vm_port(msg.get_vm_id(), msg.get_vm_port(), - msg.get_hwaddress()) - elif channel == RFSERVER_RFPROXY_CHANNEL: - if type_ == DATAPATH_PORT_REGISTER: - self.register_dp_port(msg.get_ct_id(), - msg.get_dp_id(), - msg.get_dp_port()) - elif type_ == DATAPATH_DOWN: - self.set_dp_down(msg.get_ct_id(), msg.get_dp_id()) - elif type_ == VIRTUAL_PLANE_MAP: - self.map_port(msg.get_vm_id(), msg.get_vm_port(), - msg.get_vs_id(), msg.get_vs_port()) - elif type_ == ROUTE_MOD: - self.send_routemod_acks() + if type_ == PORT_REGISTER: + self.register_vm_port(msg.get_vm_id(), msg.get_vm_port(), + msg.get_hwaddress()) + elif type_ == ROUTE_MOD: + self.register_route_mod(msg) + elif type_ == DATAPATH_PORT_REGISTER: + self.register_dp_port(msg.get_ct_id(), + msg.get_dp_id(), + msg.get_dp_port()) + elif type_ == DATAPATH_DOWN: + self.set_dp_down(msg.get_ct_id(), msg.get_dp_id()) + elif type_ == VIRTUAL_PLANE_MAP: + self.map_port(msg.get_vm_id(), msg.get_vm_port(), + msg.get_vs_id(), msg.get_vs_port()) # Port register methods def register_vm_port(self, vm_id, vm_port, eth_addr): + + entry = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) + if entry is not None: + entry.update_eth_addr(eth_addr = eth_addr) + else: + entry = RFVMPortEntry(vm_id=vm_id, vm_port=vm_port, eth_addr=eth_addr) + + self.vmporttable.set_entry(entry) + self.log.info("Updating vm port table (vm_id=%s, vm, vm_port=%i, eth_addr=%s)" + % (format(vm_id), vm_port, eth_addr)) + entry = None action = None config_entry = self.config.get_config_for_vm_port(vm_id, vm_port) if config_entry is None: @@ -567,6 +713,8 @@ def register_vm_port(self, vm_id, vm_port, eth_addr): action = REGISTER_IDLE self.log.warning('No config entry for client port (vm_id=%s, vm_port=%i)' % (format_id(vm_id), vm_port)) + # Shouldn't do anything if not configured + return else: entry = self.rftable.get_entry_by_dp_port(config_entry.ct_id, config_entry.dp_id, @@ -593,14 +741,16 @@ def register_vm_port(self, vm_id, vm_port, eth_addr): "eth_addr = %s, dp_id=%s, dp_port=%s)" % (format_id(vm_id), vm_port, eth_addr, format_id(entry.dp_id), entry.dp_port)) + - def queue_routemod_ack(self, ct_id, vm_id, vm_port): - self.ack_q.put((str(vm_id), - PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK))) + def acknowledge_route_mod(self, ct_id, vm_id, vm_port): + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), + PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK)) + return def send_route_mod(self, ct_id, rm): rm.add_option(Option.CT_ID(ct_id)) - self.dp_q.put((str(ct_id), rm)) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(ct_id), rm) # Handle RouteMod messages (type ROUTE_MOD) # @@ -609,10 +759,9 @@ def send_route_mod(self, ct_id, rm): def register_route_mod(self, rm): vm_id = rm.get_id() vm_port = rm.get_vm_port() - + # Find the (vmid, vm_port), (dpid, dpport) pair entry = self.rftable.get_entry_by_vm_port(vm_id, vm_port) - translator = self.route_mod_translator[entry.dp_id] # If we can't find an associated datapath for this RouteMod, # drop it. @@ -622,6 +771,8 @@ def register_route_mod(self, rm): (format_id(vm_id), vm_port)) return + translator = self.route_mod_translator[entry.dp_id] + # Replace the VM id,port with the Datapath id.port rm.set_id(int(entry.dp_id)) @@ -646,8 +797,7 @@ def register_route_mod(self, rm): for rm in rms: self.send_route_mod(entry.ct_id, rm) - - self.queue_routemod_ack(entry.ct_id, vm_id, vm_port) + self.acknowledge_route_mod(entry.ct_id, vm_id, vm_port) # DatapathPortRegister methods def register_dp_port(self, ct_id, dp_id, dp_port): @@ -656,6 +806,10 @@ def register_dp_port(self, ct_id, dp_id, dp_port): return # The logic down here is pretty much the same as register_vm_port + entry = RFDPPortEntry(ct_id = ct_id, dp_id = dp_id, dp_port = dp_port) + self.dpporttable.set_entry(entry) + + entry = None action = None config_entry = self.config.get_config_for_dp_port(ct_id, dp_id, dp_port) @@ -666,6 +820,8 @@ def register_dp_port(self, ct_id, dp_id, dp_port): else: # Register idle DP awaiting for configuration action = REGISTER_IDLE + # Shouldn't do anything if not configured + return else: entry = self.rftable.get_entry_by_vm_port(config_entry.vm_id, config_entry.vm_port) @@ -682,8 +838,24 @@ def register_dp_port(self, ct_id, dp_id, dp_port): dp_port=dp_port)) self.log.info("Registering datapath port as idle (dp_id=%s, " "dp_port=%i)" % (format_id(dp_id), dp_port)) - elif action == REGISTER_ASSOCIATED: + elif action == REGISTER_ASSOCIATED: entry.associate(dp_id, dp_port, ct_id) + + # We can ACTIVATE a mapping with info from VM ports table + # without the need of PortMap msg + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=entry.vm_id, vm_port=entry.vm_port) + if vm_port_info is not None and vm_port_info.vs_id is not None: + entry.activate(vs_id=vm_port_info.vs_id, + vs_port=vm_port_info.vs_port) + + msg = DataPlaneMap(ct_id=entry.ct_id, + dp_id=entry.dp_id, dp_port=entry.dp_port, + vs_id=entry.vs_id, vs_port=entry.vs_port, operation_id=DCT_UPDATE_DP) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + msg = PortConfig(vm_id=entry.vm_id, vm_port=entry.vm_port, + operation_id=PCT_MAP_SUCCESS) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) + self.rftable.set_entry(entry) self.log.info("Registering datapath port and associating to " "client port (dp_id=%s, dp_port=%i, vm_id=%s, " @@ -693,6 +865,7 @@ def register_dp_port(self, ct_id, dp_id, dp_port): elif action == REGISTER_ISL: self._register_islconf(islconfs, ct_id, dp_id, dp_port) + def _register_islconf(self, c_entries, ct_id, dp_id, dp_port): for conf in c_entries: entry = None @@ -748,7 +921,7 @@ def _register_islconf(self, c_entries, ct_id, dp_id, dp_port): def send_datapath_config_messages(self, ct_id, dp_id): rms = self.route_mod_translator[dp_id].configure_datapath() for rm in rms: - self.send_route_mod(ct_id, rm) + self.send_route_mod(ct_id, rm) def config_dp(self, ct_id, dp_id): if is_rfvs(dp_id): @@ -771,6 +944,9 @@ def config_dp(self, ct_id, dp_id): return False # DatapathDown methods def set_dp_down(self, ct_id, dp_id): + for entry in self.dpporttable.get_entries(dp_id=dp_id): + self.dpporttable.remove_entry(entry) + for entry in self.rftable.get_dp_entries(ct_id, dp_id): # For every port registered in that datapath, put it down self.set_dp_port_down(entry.ct_id, entry.dp_id, entry.dp_port) @@ -798,7 +974,7 @@ def set_dp_port_down(self, ct_id, dp_id, dp_port): def reset_vm_port(self, vm_id, vm_port): if vm_id is None: return - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_RESET)) self.log.info("Resetting client port (vm_id=%s, vm_port=%i)" % @@ -806,6 +982,16 @@ def reset_vm_port(self, vm_id, vm_port): # PortMap methods def map_port(self, vm_id, vm_port, vs_id, vs_port): + + # Check if the port is already in VM Ports Table, Update the port entry if found + entry = self.vmporttable.get_vm_port_info(vm_id = vm_id, vm_port = vm_port) + if entry is not None: + entry.update_vs(vs_id = vs_id, vs_port = vs_port) + self.vmporttable.set_entry(entry) + #TODO: Because we keep VS info, so no need RFClient to send this msg anymore + # When register dp port, we can get vm and vs info to update rftable + + entry = None entry = self.rftable.get_entry_by_vm_port(vm_id, vm_port) if entry is not None and entry.get_status() == RFENTRY_ASSOCIATED: # If the association is valid, activate it @@ -813,18 +999,19 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): self.rftable.set_entry(entry) msg = DataPlaneMap(ct_id=entry.ct_id, dp_id=entry.dp_id, dp_port=entry.dp_port, - vs_id=vs_id, vs_port=vs_port) - self.ipc_send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + vs_id=vs_id, vs_port=vs_port, operation_id=DCT_UPDATE_DP) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) msg = PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_MAP_SUCCESS) - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) self.log.info("Mapping client-datapath association " "(vm_id=%s, vm_port=%i, dp_id=%s, " "dp_port=%i, vs_id=%s, vs_port=%i)" % (format_id(entry.vm_id), entry.vm_port, format_id(entry.dp_id), entry.dp_port, format_id(entry.vs_id), entry.vs_port)) - + + if __name__ == "__main__": description = 'RFServer co-ordinates RFClient and RFProxy instances, ' \ 'listens for route updates, and configures flow tables' @@ -844,4 +1031,7 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): help='List of datapaths that default forward to ISL peer') args = parser.parse_args() - server = RFServer(args.configfile, args.islconfig, args.multitabledps, args.satellitedps) + rfserver = RFServer(args.configfile, args.islconfig, args.multitabledps, args.satellitedps) + + #RPC interface + rpcserver = RFServerRPC(rfserver) \ No newline at end of file diff --git a/rfserver/rftable.py b/rfserver/rftable.py index 71710ec3..08f654a4 100644 --- a/rfserver/rftable.py +++ b/rfserver/rftable.py @@ -18,6 +18,9 @@ RFISLCONFENTRY = 2 RFISLENTRY = 3 +RFVMPORTENTRY = 4 +RFDPPORTENTRY = 5 + class EntryFactory: @staticmethod def make(type_): @@ -29,6 +32,10 @@ def make(type_): return RFISLEntry() elif type_ == RFISLCONFENTRY: return RFISLConfEntry() + elif type_ == RFVMPORTENTRY: + return RFVMPortEntry() + elif type_ == RFDPPORTENTRY: + return RFDPPortEntry(); class EntryTable(TableBase): def __init__(self, name, entry_type): @@ -195,7 +202,21 @@ def _is_idle_vm_port(self): self.dp_port is None and self.vs_id is None and self.vs_port is None) - + + # Used for dynamic mapping to update dp port info of an entry + def update_dp_port(self, ct_id = None,dp_id=None, dp_port=None): + self.ct_id = ct_id + self.dp_id = dp_id + self.dp_port = dp_port + + # Used for dynamic mapping to update vm port info of an entry + def update_vm_port(self, vm_id = None, vm_port=None, vs_id=None, vs_port=None, eth_addr=None): + self.vm_id = vm_id + self.vm_port = vm_port + self.vs_id = vs_id + self.vs_port = vs_port + self.eth_addr = eth_addr + def _is_idle_dp_port(self): return (self.vm_id is None and self.vm_port is None and @@ -450,7 +471,18 @@ def __init__(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, self.ct_id = ct_id self.dp_id = dp_id self.dp_port = dp_port - + + # Used for dynamic mapping to update dp port info of an entry + def update_dp_port(self, ct_id, dp_id, dp_port): + self.ct_id = ct_id + self.dp_id = dp_id + self.dp_port = dp_port + + # Used for dynamic mapping to update vm port info of an entry + def update_vm_port(self, vm_id, vm_port): + self.vm_id = vm_id + self.vm_port = vm_port + def __str__(self): return "vm_id: %s vm_port: %s "\ "dp_id: %s dp_port: %s "\ @@ -476,3 +508,85 @@ def to_dict(self): pack_into_dict(data, self, "dp_id") pack_into_dict(data, self, "dp_port") return data + +class RFVMPortEntry: + def __init__(self, id=None, vm_id=None, vm_port=None, eth_addr=None, vs_id=None, vs_port=None, state=None): + self.id = id + self.vm_id = vm_id + self.vm_port = vm_port + self.eth_addr = eth_addr + self.vs_id = vs_id + self.vs_port = vs_port + self.state = state + + def update_vs(self, vs_id, vs_port): + self.vs_id = vs_id + self.vs_port = vs_port + + def update_eth_addr(self, eth_addr): + self.eth_addr = eth_add + + def to_dict(self): + data = {} + if self.id is not None: + data["_id"] = self.id + pack_into_dict(data, self, "vm_id") + pack_into_dict(data, self, "vm_port") + pack_into_dict(data, self, "vs_id") + pack_into_dict(data, self, "vs_port") + pack_into_dict(data, self, "eth_addr") + return data + + def from_dict(self, data): + self.id = data["_id"] + load_from_dict(data, self, "vm_id") + load_from_dict(data, self, "vm_port") + load_from_dict(data, self, "eth_addr") + load_from_dict(data, self, "vs_id") + load_from_dict(data, self, "vs_port") + +class RFVMPortTable(EntryTable): + def __init__(self): + EntryTable.__init__(self, RFVMPORTTABLE_NAME, RFVMPORTENTRY) + + def get_vm_port_info(self, vm_id, vm_port): + result = self.get_entries(vm_id=vm_id, + vm_port=vm_port) + if not result: + return None + return result[0] + + +class RFDPPortTable(EntryTable): + def __init__(self): + EntryTable.__init__(self, RFDPPORTTABLE_NAME, RFDPPORTENTRY) + + def get_dp_port_info(self, ct_id, dp_id, dp_port): + result = self.get_entries(ct_id= ct_id, dp_id=dp_id, + dp_port=dp_port) + if not result: + return None + return result[0] + +class RFDPPortEntry: + def __init__(self, id=None, ct_id=None, dp_id=None, dp_port=None, state=None): + self.id = id + self.ct_id = ct_id + self.dp_id = dp_id + self.dp_port = dp_port + self.state = state + + def to_dict(self): + data = {} + if self.id is not None: + data["_id"] = self.id + pack_into_dict(data, self, "ct_id") + pack_into_dict(data, self, "dp_port") + pack_into_dict(data, self, "dp_id") + return data + + def from_dict(self, data): + self.id = data["_id"] + load_from_dict(data, self, "ct_id") + load_from_dict(data, self, "dp_port") + load_from_dict(data, self, "dp_id") \ No newline at end of file From 158b4ba867f06947f360425656e60695044bfde9 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 7 Jan 2015 13:30:00 +1300 Subject: [PATCH 15/52] Support dynamic mapping Initial push as for backup Details of change will be explained later Signed-off-by: Trung Truong --- rflib/defs.py | 13 +- rflib/ipc/RFProtocol.py | 17 +- rfserver/rfserver.py | 346 +++++++++++++++++++++++++++++++--------- rfserver/rfservercli.py | 212 ++++++++++++++++++++++++ rfserver/rfserverrpc.py | 58 +++++++ rfserver/rftable.py | 118 +++++++++++++- 6 files changed, 681 insertions(+), 83 deletions(-) create mode 100755 rfserver/rfservercli.py create mode 100755 rfserver/rfserverrpc.py diff --git a/rflib/defs.py b/rflib/defs.py index d41d7bcb..a58eb845 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -2,7 +2,7 @@ IPC_TYPE = 'zeromq' # options are 'zeromq' or 'mongo' DB_TYPE = 'memory' # options are 'memory' or 'mongo' -MONGO_ADDRESS = "192.168.10.1:27017" +MONGO_ADDRESS = "127.0.0.1:27017" MONGO_DB_NAME = "db" ZEROMQ_ADDRESS = "tcp://192.168.10.1:25555" @@ -15,6 +15,9 @@ RFISL_NAME = "rfisl" RFISLCONF_NAME = "rfislconf" +RFVMPORTTABLE_NAME = "vmporttable" +RFDPPORTTABLE_NAME = "dpporttable" + RFSERVER_ID = "rfserver" RFPROXY_ID = "rfproxy" @@ -64,8 +67,14 @@ PC_MAP = 0 PC_RESET = 1 +# Data plane Port Map Configuration (DCT_) +DCT_UPDATE_DP = 0 # Update Proxy map table by DP info +DCT_UPDATE_VS = 1 # Update Proxy map table by VS info +DCT_DELETE_DP = 2 # Delete Proxy map table by DP info +DCT_DELETE_VS = 3 # Delete Proxy map table by VS info + # Format 12-digit hex ID -format_id = lambda dp_id: hex(dp_id).rstrip("L") +format_id = lambda dp_id: hex(dp_id).rstrip("L") if (dp_id is not None) else 'None' netmask_prefix = lambda a: sum([bin(int(x)).count("1") for x in a.split(".", 4)]) cidr_to_mask = lambda a: ((1 << a) - 1) << (32 - a) diff --git a/rflib/ipc/RFProtocol.py b/rflib/ipc/RFProtocol.py index aeeb8bd2..662b6853 100644 --- a/rflib/ipc/RFProtocol.py +++ b/rflib/ipc/RFProtocol.py @@ -14,6 +14,7 @@ VIRTUAL_PLANE_MAP = 4 DATA_PLANE_MAP = 5 ROUTE_MOD = 6 +DATA_PORT_REQUEST = 7 class PortRegister(IPCMessage): def __init__(self, vm_id=None, vm_port=None, hwaddress=None): @@ -306,12 +307,24 @@ def __str__(self): return s class DataPlaneMap(IPCMessage): - def __init__(self, ct_id=None, dp_id=None, dp_port=None, vs_id=None, vs_port=None): + def __init__(self, ct_id=None, dp_id=None, dp_port=None, vs_id=None, vs_port=None, operation_id=None): self.set_ct_id(ct_id) self.set_dp_id(dp_id) self.set_dp_port(dp_port) self.set_vs_id(vs_id) self.set_vs_port(vs_port) + + self.set_operation_id(operation_id) + + def get_operation_id(self): + return self.operation_id + + def set_operation_id(self, operation_id): + operation_id = 0 if operation_id is None else operation_id + try: + self.operation_id = int(operation_id) + except: + self.operation_id = 0 def get_type(self): return DATA_PLANE_MAP @@ -372,6 +385,7 @@ def from_dict(self, data): self.set_dp_port(data["dp_port"]) self.set_vs_id(data["vs_id"]) self.set_vs_port(data["vs_port"]) + self.set_operation_id(data["operation_id"]) def to_dict(self): data = {} @@ -380,6 +394,7 @@ def to_dict(self): data["dp_port"] = str(self.get_dp_port()) data["vs_id"] = str(self.get_vs_id()) data["vs_port"] = str(self.get_vs_port()) + data["operation_id"] = str(self.get_operation_id()) return data def __str__(self): diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 76cc4f33..fd1fdece 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -9,7 +9,7 @@ import time import copy import Queue -import threading +import signal from bson.binary import Binary @@ -24,6 +24,8 @@ from rftable import * +from rfserverrpc import RFServerRPC + logging.basicConfig( level=logging.INFO, format='%(asctime)s %(name)-15s %(levelname)-8s %(message)s', @@ -35,7 +37,6 @@ REGISTER_ASSOCIATED = 1 REGISTER_ISL = 2 - class RouteModTranslator(object): DROP_PRIORITY = Option.PRIORITY(PRIORITY_LOWEST + PRIORITY_BAND) @@ -48,7 +49,7 @@ def __init__(self, dp_id, ct_id, rftable, isltable, log): self.rftable = rftable self.isltable = isltable self.log = log - + def configure_datapath(self): raise Exception @@ -108,6 +109,7 @@ def handle_controller_route_mod(self, entry, rm): def handle_route_mod(self, entry, rm): rms = [] + entries = self.rftable.get_entries(dp_id=entry.dp_id, ct_id=entry.ct_id) entries.extend(self.isltable.get_entries(dp_id=entry.dp_id, @@ -130,7 +132,17 @@ def handle_isl_route_mod(self, r, rm): entries = self.rftable.get_entries(dp_id=r.dp_id, ct_id=r.ct_id) rms.extend(self._send_rm_with_matches(rm, r.dp_port, entries)) return rms - + # This method used to remove flow entries from switches when + # users make a change (i.e. delete) of an association + def dp_flows_remove_by_vm_port(self, entry): + rms = [] + rm = RouteMod(RMT_DELETE, self.dp_id) + rm.add_match(Match.IN_PORT(entry.dp_port)) + rm.add_match(Match.ETHERNET(entry.eth_addr)) + rm.add_option(self.DEFAULT_PRIORITY) + rms.append(rm) + print rm + return rms class SatelliteRouteModTranslator(DefaultRouteModTranslator): @@ -158,8 +170,10 @@ def handle_isl_route_mod(self, r, rm): rms.extend(self._send_rm_with_matches(rm, r.dp_port, entries)) return rms - class NoviFlowMultitableRouteModTranslator(RouteModTranslator): + # NoviFlow as of Aug 2014 doesn't handle flow add operations + # in unsorted priority order well (very slow). For the moment + # make all flows in a table same priority. FIB_TABLE = 2 ETHER_TABLE = 1 @@ -260,19 +274,20 @@ def handle_route_mod(self, entry, rm): rm.add_action(Action.OUTPUT(entry.dp_port)) rm.set_table(self.FIB_TABLE) - + # See NoviFlow note rm.set_options(None) - rm.add_option(self.CONTROLLER_PRIORITY) + rm.add_option(Option.PRIORITY(PRIORITY_HIGH)) rms.extend(self._send_rm_with_matches(rm, entry.dp_port, entries)) return rms def handle_isl_route_mod(self, r, rm): rms = [] + # See NoviFlow note + rm.set_options(None) + rm.add_option(Option.PRIORITY(PRIORITY_HIGH)) rm.set_id(self.dp_id) rm.set_table(self.FIB_TABLE) - rm.set_options(None) - rm.add_option(self.CONTROLLER_PRIORITY) rm.set_actions(None) rm.add_action(Action.SET_ETH_SRC(r.eth_addr)) rm.add_action(Action.SET_ETH_DST(r.rem_eth_addr)) @@ -389,17 +404,12 @@ def _send_rm_with_matches(self, rm, out_port, entries): if (entry.get_status() == RFENTRY_ACTIVE or entry.get_status() == RFISL_ACTIVE): dst_eth = None - actions = rm.actions - rm.set_actions(None) - for action_dict in actions: + for action_dict in rm.actions: action = Action.from_dict(action_dict) action_type = action.type_to_str(action._type) if action_type == 'RFAT_SET_ETH_DST': dst_eth = action.get_value() - elif action_type == 'RFAT_SWAP_VLAN_ID': - vlan_id = action.get_value() - action = Action.SET_VLAN_ID(vlan_id) - rm.add_action(action) + break if dst_eth not in self.actions_to_groupid: self.last_groupid += 1 new_groupid = self.last_groupid @@ -478,10 +488,10 @@ def handle_route_mod(self, entry, rm): return rms class RFServer(RFProtocolFactory, IPC.IPCMessageProcessor): - def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.config = RFConfig(configfile) self.islconf = RFISLConf(islconffile) + try: self.multitabledps = set([int(x, 16) for x in multitabledps.split(",")]) except ValueError: @@ -494,6 +504,12 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): # Initialise state tables self.rftable = RFTable() self.isltable = RFISLTable() + + # Initalize two new tables for VM ports and DP ports state + # At this stage, the tables keep simple port info as RFTable does + #TODO: 1. Add more port state info (e.g. speed, desc, medium type...). 2. Link RFTable to these tables to avoid duplicate data + self.vmporttable = RFVMPortTable() + self.dpporttable = RFDPPortTable() self.route_mod_translator = {} @@ -507,59 +523,189 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.log.info("Datapaths that support multiple tables: %s", list(self.multitabledps)) - self.ack_q = Queue.Queue() - self.dp_q = Queue.Queue() - self.ipc_lock = threading.Lock() - self.routemod_outstanding = threading.Event() self.ipc = IPCService.for_server(RFSERVER_ID) - - self.worker = threading.Thread(target=self.dp_worker) - self.worker.daemon = True - self.worker.start() - self.ipc.listen(RFCLIENT_RFSERVER_CHANNEL, self, self, False) - self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, True) - - def ipc_send(self, channel, channel_id, msg): - self.ipc_lock.acquire() - self.ipc.send(channel, channel_id, msg) - self.ipc_lock.release() + self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, False) + + # Delete a mapping configuration. + def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + cf_entry = None + if vm_id is not None and vm_port is not None: + cf_entry = self.config.get_config_for_vm_port(vm_id, vm_port) + + elif dp_id is not None and dp_port is not None and ct_id is not None: + cf_entry = self.config.get_config_for_dp_port(ct_id, dp_id, dp_port) + + if cf_entry is not None: #Delete it from config table + self.config.remove_entry(cf_entry) + # Send reset signal to RFClient, so it starts generating PortMap msg + #self.reset_vm_port(cf_entry.vm_id, cf_entry.vm_port) + # Update map table in RFProxy + rf_entry = self.rftable.get_entry_by_vm_port(cf_entry.vm_id, cf_entry.vm_port) + if rf_entry is not None: + # Delete from RFTable & send a reset msg to RFProxy + self.rftable.remove_entry(rf_entry) + if rf_entry.get_status() == RFENTRY_ACTIVE: + msg = DataPlaneMap(ct_id=rf_entry.ct_id, + dp_id=rf_entry.dp_id, dp_port=rf_entry.dp_port, + vs_id=rf_entry.vs_id, vs_port=rf_entry.vs_port, operation_id=DCT_DELETE_DP) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(rf_entry.ct_id), msg) + self.log.info("Resetting RFProxy map table (dp_id=%s, dp_port=%i) - (vs_id=%s, vs_port=%i)" % + (format_id(rf_entry.dp_id), rf_entry.dp_port, format_id(rf_entry.vs_id), rf_entry.vs_port)) + #TODO: Update flow table of related dp + translator = self.route_mod_translator[rf_entry.dp_id] + rms = translator.dp_flows_remove_by_vm_port(rf_entry) + for rm in rms: + self.send_route_mod(rf_entry.ct_id, rm) + return True + + self.log.info("No config found") + return False + + # Update or create a mapping. If not exist, create it. + # A VM port can be mapped to a new DP port if the DP port is not being used, similar to DP port mapping change. + def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + if vm_id is None or vm_port is None \ + or dp_id is None or dp_port is None or ct_id is None: + return False + + vm_port_conf = self.config.get_config_for_vm_port(vm_id=vm_id, vm_port=vm_port) + dp_port_conf = self.config.get_config_for_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + old_conf = None + rf_entry = None + action = None + # 1. Update CONFIG table + if vm_port_conf is not None: + if dp_port_conf is not None: + # Both are in use, so do nothing + self.log.info("Both VM port and DP port are in used. No update can be done") + return False + else: # Map VM port to a new DP port + vm_port_conf.update_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + self.config.set_entry(vm_port_conf) + action = 1 + else: # Map DP port to a new VM port + if dp_port_conf is not None: # Map DP port to a new VM port + dp_port_conf.update_vm_port(vm_id=vm_id, vm_port=vm_port) + self.config.set_entry(dp_port_conf) + action = 2 + else: # Create a new mapping between VM port & DP port + self.config.set_entry(RFConfigEntry(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port)) + + action = 3 + + # 2. Update RFTable + rfclient_inform = False + if action == 1: # If update mapping of VM port + rf_entry = self.rftable.get_entry_by_vm_port(vm_id=vm_id, vm_port=vm_port) + if rf_entry is None: + rf_entry = RFEntry() + dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if dp_port_info is not None: + rf_entry.update_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if rf_entry.get_status() == RFENTRY_ACTIVE: + rfproxy_oper_id = DCT_UPDATE_VS + rfclient_inform = True + else: # We have no info about the new dp port + rf_entry.make_idle(RFENTRY_IDLE_VM_PORT) + # Reset rfproxy table + rfproxy_oper_id = DCT_DELETE_VS + # Update to RFTable + self.rftable.set_entry(rf_entry) + elif action == 2: # Update mapping of DP port + rf_entry = self.rftable.get_entry_by_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if rf_entry is None: + rf_entry = RFEntry() + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) + if vm_port_info is not None: + rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, + vs_id=vm_port_info.vs_id, + vs_port=vm_port_info.vs_port, + eth_addr=vm_port_info.eth_addr) + self.rftable.set_entry(rf_entry) + if rf_entry.get_status() == RFENTRY_ACTIVE: + rfproxy_oper_id = DCT_UPDATE_DP + rfclient_inform = True + else: # We have no info about the new vm port + rf_entry.make_idle(RFENTRY_IDLE_DP_PORT) + self.rftable.set_entry(rf_entry) + rfproxy_oper_id = DCT_DELETE_DP + else: # Create a new mapping + rf_entry = RFEntry() + dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) + + if dp_port_info is not None: + rf_entry.update_dp_port(dp_id=dp_id, dp_port=dp_port, ct_id=ct_id) + + if vm_port_info is not None: + rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, + vs_id=vm_port_info.vs_id, + vs_port=vm_port_info.vs_port, + eth_addr=vm_port_info.eth_addr) + + if vm_port_info is None and dp_port_info is None: + rfproxy_oper_id = None + else: + self.rftable.set_entry(rf_entry) + if rf_entry.get_status() == RFENTRY_ACTIVE: + rfproxy_oper_id = DCT_UPDATE_DP + rfclient_inform = True + + if rfproxy_oper_id is not None: + msg = DataPlaneMap(ct_id=rf_entry.ct_id, + dp_id=rf_entry.dp_id, + dp_port=rf_entry.dp_port, + vs_id=rf_entry.vs_id, + vs_port=rf_entry.vs_port, + operation_id=rfproxy_oper_id) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(rf_entry.ct_id), msg) + + if rfclient_inform: + # Inform RFClient about the new mapping + msg = PortConfig(vm_id=vm_id, vm_port=vm_port, + operation_id=PCT_MAP_SUCCESS) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(rf_entry.vm_id), msg) + + self.log.info("Updating a new client-datapath association " + "(vm_id=%s, vm_port=%s, dp_id=%s, " + "dp_port=%s, vs_id=%s, vs_port=%s)" % + (format_id(rf_entry.vm_id), rf_entry.vm_port, + format_id(rf_entry.dp_id), rf_entry.dp_port, + format_id(rf_entry.vs_id), rf_entry.vs_port)) + return True - def dp_worker(self): - while True: - (ct_id, rm) = self.dp_q.get(block=True) - self.ipc_send(RFSERVER_RFPROXY_CHANNEL, ct_id, rm) - self.dp_q.task_done() - - def send_routemod_acks(self): - while not self.ack_q.empty(): - (vm_id, ack) = self.ack_q.get() - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, vm_id, ack) - self.ack_q.task_done() - def process(self, from_, to, channel, msg): type_ = msg.get_type() - if channel == RFCLIENT_RFSERVER_CHANNEL: - if type_ == ROUTE_MOD: - self.register_route_mod(msg) - elif type_ == PORT_REGISTER: - self.register_vm_port(msg.get_vm_id(), msg.get_vm_port(), - msg.get_hwaddress()) - elif channel == RFSERVER_RFPROXY_CHANNEL: - if type_ == DATAPATH_PORT_REGISTER: - self.register_dp_port(msg.get_ct_id(), - msg.get_dp_id(), - msg.get_dp_port()) - elif type_ == DATAPATH_DOWN: - self.set_dp_down(msg.get_ct_id(), msg.get_dp_id()) - elif type_ == VIRTUAL_PLANE_MAP: - self.map_port(msg.get_vm_id(), msg.get_vm_port(), - msg.get_vs_id(), msg.get_vs_port()) - elif type_ == ROUTE_MOD: - self.send_routemod_acks() + if type_ == PORT_REGISTER: + self.register_vm_port(msg.get_vm_id(), msg.get_vm_port(), + msg.get_hwaddress()) + elif type_ == ROUTE_MOD: + self.register_route_mod(msg) + elif type_ == DATAPATH_PORT_REGISTER: + self.register_dp_port(msg.get_ct_id(), + msg.get_dp_id(), + msg.get_dp_port()) + elif type_ == DATAPATH_DOWN: + self.set_dp_down(msg.get_ct_id(), msg.get_dp_id()) + elif type_ == VIRTUAL_PLANE_MAP: + self.map_port(msg.get_vm_id(), msg.get_vm_port(), + msg.get_vs_id(), msg.get_vs_port()) # Port register methods def register_vm_port(self, vm_id, vm_port, eth_addr): + + entry = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) + if entry is not None: + entry.update_eth_addr(eth_addr = eth_addr) + else: + entry = RFVMPortEntry(vm_id=vm_id, vm_port=vm_port, eth_addr=eth_addr) + + self.vmporttable.set_entry(entry) + self.log.info("Updating vm port table (vm_id=%s, vm, vm_port=%i, eth_addr=%s)" + % (format(vm_id), vm_port, eth_addr)) + entry = None action = None config_entry = self.config.get_config_for_vm_port(vm_id, vm_port) if config_entry is None: @@ -567,6 +713,8 @@ def register_vm_port(self, vm_id, vm_port, eth_addr): action = REGISTER_IDLE self.log.warning('No config entry for client port (vm_id=%s, vm_port=%i)' % (format_id(vm_id), vm_port)) + # Shouldn't do anything if not configured + return else: entry = self.rftable.get_entry_by_dp_port(config_entry.ct_id, config_entry.dp_id, @@ -593,14 +741,16 @@ def register_vm_port(self, vm_id, vm_port, eth_addr): "eth_addr = %s, dp_id=%s, dp_port=%s)" % (format_id(vm_id), vm_port, eth_addr, format_id(entry.dp_id), entry.dp_port)) + - def queue_routemod_ack(self, ct_id, vm_id, vm_port): - self.ack_q.put((str(vm_id), - PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK))) + def acknowledge_route_mod(self, ct_id, vm_id, vm_port): + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), + PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_ROUTEMOD_ACK)) + return def send_route_mod(self, ct_id, rm): rm.add_option(Option.CT_ID(ct_id)) - self.dp_q.put((str(ct_id), rm)) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(ct_id), rm) # Handle RouteMod messages (type ROUTE_MOD) # @@ -609,10 +759,9 @@ def send_route_mod(self, ct_id, rm): def register_route_mod(self, rm): vm_id = rm.get_id() vm_port = rm.get_vm_port() - + # Find the (vmid, vm_port), (dpid, dpport) pair entry = self.rftable.get_entry_by_vm_port(vm_id, vm_port) - translator = self.route_mod_translator[entry.dp_id] # If we can't find an associated datapath for this RouteMod, # drop it. @@ -622,6 +771,8 @@ def register_route_mod(self, rm): (format_id(vm_id), vm_port)) return + translator = self.route_mod_translator[entry.dp_id] + # Replace the VM id,port with the Datapath id.port rm.set_id(int(entry.dp_id)) @@ -646,8 +797,7 @@ def register_route_mod(self, rm): for rm in rms: self.send_route_mod(entry.ct_id, rm) - - self.queue_routemod_ack(entry.ct_id, vm_id, vm_port) + self.acknowledge_route_mod(entry.ct_id, vm_id, vm_port) # DatapathPortRegister methods def register_dp_port(self, ct_id, dp_id, dp_port): @@ -656,6 +806,10 @@ def register_dp_port(self, ct_id, dp_id, dp_port): return # The logic down here is pretty much the same as register_vm_port + entry = RFDPPortEntry(ct_id = ct_id, dp_id = dp_id, dp_port = dp_port) + self.dpporttable.set_entry(entry) + + entry = None action = None config_entry = self.config.get_config_for_dp_port(ct_id, dp_id, dp_port) @@ -666,6 +820,8 @@ def register_dp_port(self, ct_id, dp_id, dp_port): else: # Register idle DP awaiting for configuration action = REGISTER_IDLE + # Shouldn't do anything if not configured + return else: entry = self.rftable.get_entry_by_vm_port(config_entry.vm_id, config_entry.vm_port) @@ -682,8 +838,24 @@ def register_dp_port(self, ct_id, dp_id, dp_port): dp_port=dp_port)) self.log.info("Registering datapath port as idle (dp_id=%s, " "dp_port=%i)" % (format_id(dp_id), dp_port)) - elif action == REGISTER_ASSOCIATED: + elif action == REGISTER_ASSOCIATED: entry.associate(dp_id, dp_port, ct_id) + + # We can ACTIVATE a mapping with info from VM ports table + # without the need of PortMap msg + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=entry.vm_id, vm_port=entry.vm_port) + if vm_port_info is not None and vm_port_info.vs_id is not None: + entry.activate(vs_id=vm_port_info.vs_id, + vs_port=vm_port_info.vs_port) + + msg = DataPlaneMap(ct_id=entry.ct_id, + dp_id=entry.dp_id, dp_port=entry.dp_port, + vs_id=entry.vs_id, vs_port=entry.vs_port, operation_id=DCT_UPDATE_DP) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + msg = PortConfig(vm_id=entry.vm_id, vm_port=entry.vm_port, + operation_id=PCT_MAP_SUCCESS) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) + self.rftable.set_entry(entry) self.log.info("Registering datapath port and associating to " "client port (dp_id=%s, dp_port=%i, vm_id=%s, " @@ -693,6 +865,7 @@ def register_dp_port(self, ct_id, dp_id, dp_port): elif action == REGISTER_ISL: self._register_islconf(islconfs, ct_id, dp_id, dp_port) + def _register_islconf(self, c_entries, ct_id, dp_id, dp_port): for conf in c_entries: entry = None @@ -748,7 +921,7 @@ def _register_islconf(self, c_entries, ct_id, dp_id, dp_port): def send_datapath_config_messages(self, ct_id, dp_id): rms = self.route_mod_translator[dp_id].configure_datapath() for rm in rms: - self.send_route_mod(ct_id, rm) + self.send_route_mod(ct_id, rm) def config_dp(self, ct_id, dp_id): if is_rfvs(dp_id): @@ -771,6 +944,9 @@ def config_dp(self, ct_id, dp_id): return False # DatapathDown methods def set_dp_down(self, ct_id, dp_id): + for entry in self.dpporttable.get_entries(dp_id=dp_id): + self.dpporttable.remove_entry(entry) + for entry in self.rftable.get_dp_entries(ct_id, dp_id): # For every port registered in that datapath, put it down self.set_dp_port_down(entry.ct_id, entry.dp_id, entry.dp_port) @@ -798,7 +974,7 @@ def set_dp_port_down(self, ct_id, dp_id, dp_port): def reset_vm_port(self, vm_id, vm_port): if vm_id is None: return - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(vm_id), PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_RESET)) self.log.info("Resetting client port (vm_id=%s, vm_port=%i)" % @@ -806,6 +982,16 @@ def reset_vm_port(self, vm_id, vm_port): # PortMap methods def map_port(self, vm_id, vm_port, vs_id, vs_port): + + # Check if the port is already in VM Ports Table, Update the port entry if found + entry = self.vmporttable.get_vm_port_info(vm_id = vm_id, vm_port = vm_port) + if entry is not None: + entry.update_vs(vs_id = vs_id, vs_port = vs_port) + self.vmporttable.set_entry(entry) + #TODO: Because we keep VS info, so no need RFClient to send this msg anymore + # When register dp port, we can get vm and vs info to update rftable + + entry = None entry = self.rftable.get_entry_by_vm_port(vm_id, vm_port) if entry is not None and entry.get_status() == RFENTRY_ASSOCIATED: # If the association is valid, activate it @@ -813,18 +999,19 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): self.rftable.set_entry(entry) msg = DataPlaneMap(ct_id=entry.ct_id, dp_id=entry.dp_id, dp_port=entry.dp_port, - vs_id=vs_id, vs_port=vs_port) - self.ipc_send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + vs_id=vs_id, vs_port=vs_port, operation_id=DCT_UPDATE_DP) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) msg = PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_MAP_SUCCESS) - self.ipc_send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) self.log.info("Mapping client-datapath association " "(vm_id=%s, vm_port=%i, dp_id=%s, " "dp_port=%i, vs_id=%s, vs_port=%i)" % (format_id(entry.vm_id), entry.vm_port, format_id(entry.dp_id), entry.dp_port, format_id(entry.vs_id), entry.vs_port)) - + + if __name__ == "__main__": description = 'RFServer co-ordinates RFClient and RFProxy instances, ' \ 'listens for route updates, and configures flow tables' @@ -844,4 +1031,7 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): help='List of datapaths that default forward to ISL peer') args = parser.parse_args() - server = RFServer(args.configfile, args.islconfig, args.multitabledps, args.satellitedps) + rfserver = RFServer(args.configfile, args.islconfig, args.multitabledps, args.satellitedps) + + #RPC interface + rpcserver = RFServerRPC(rfserver) \ No newline at end of file diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py new file mode 100755 index 00000000..2f89209f --- /dev/null +++ b/rfserver/rfservercli.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +import logging +from cliff.app import App +from cliff.commandmanager import CommandManager +from cliff.command import Command + +import xmlrpclib +import json +import sys + +import rflib.defs as defs + +def format_id(value): + try: + value = int(value) + return defs.format_id(value) + except ValueError, TypeError: + return value + +class DeleteCommand(Command): + log = logging.getLogger(__name__) + + def get_parser(self, prog_name): + parser = super(DeleteCommand, self).get_parser(prog_name) + parser.add_argument('-vm_id', '--vm_id', type=str, required=False, default=None) + parser.add_argument('-vm_port', '--vm_port', type=str, required=False, default=None) + parser.add_argument('-dp_id', '--dp_id', type=str, required=False, default=None) + parser.add_argument('-dp_port', '--dp_port', type=str, required=False, default=None) + parser.add_argument('-ct_id', '--ct_id', type=str, required=False, default=0) + + return parser + + def take_action(self, parsed_args): + vm_id = None if parsed_args.vm_id is None else int(str(parsed_args.vm_id), 16) + vm_port = None if parsed_args.vm_port is None else int(str(parsed_args.vm_port)) + + dp_id = None if parsed_args.dp_id is None else int(str(parsed_args.dp_id), 16) + dp_port = None if parsed_args.dp_port is None else int(str(parsed_args.dp_port)) + ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) + + rfserver = self.app.rfserver + result = rfserver.delete_single_mapping(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if result: + self.app.log.info("Delete operation successful") + else: + self.app.log.info("Delete operation failed") + +class UpdateCommand(Command): + log = logging.getLogger(__name__) + + def get_parser(self, prog_name): + parser = super(UpdateCommand, self).get_parser(prog_name) + parser.add_argument('-vm_id', '--vm_id', type=str, required=True, default=None) + parser.add_argument('-vm_port', '--vm_port', type=str, required=True, default=None) + parser.add_argument('-dp_id', '--dp_id', type=str, required=True, default=None) + parser.add_argument('-dp_port', '--dp_port', type=str, required=True, default=None) + parser.add_argument('-ct_id', '--ct_id', type=str, required=False, default=0) + + return parser + + def take_action(self, parsed_args): + vm_id = None if parsed_args.vm_id is None else int(str(parsed_args.vm_id), 16) + vm_port = None if parsed_args.vm_port is None else int(str(parsed_args.vm_port)) + + dp_id = None if parsed_args.dp_id is None else int(str(parsed_args.dp_id), 16) + dp_port = None if parsed_args.dp_port is None else int(str(parsed_args.dp_port)) + ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) + + rfserver = self.app.rfserver + result = rfserver.update_single_mapping(vm_id=vm_id, vm_port=vm_port, ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + if result: + self.app.log.info("Delete operation successful") + else: + self.app.log.info("Delete operation failed") + +class ViewCommand(Command): + def get_parser(self, prog_name): + parser = super(ViewCommand, self).get_parser(prog_name) + parser.add_argument('view', nargs='?', default='rftable') + parser.add_argument('-filter', '--filter', type=str, required=False, default=None) + + return parser + + def take_action(self, parsed_args): + view = str(parsed_args.view) + filter = str(parsed_args.filter) + + if view == 'rftable': + self.view_rftable() + elif view == 'isltable': + self.view_isltable(); + elif view == 'config': + self.view_config() + elif view == 'vmport': + self.view_vmports() + elif view == 'dpport': + self.view_dpports() + elif view == "islconfig": + self.view_islconfig() + else: + self.app.stdout.write("Unknown option\n") + + def view_rftable(self, **kwargs): + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rftable()) + + self.app.stdout.write("{:<18} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18} {:<8}\n\n".\ + format('vm_id', 'vm_port', 'eth_addr',\ + 'ct_id', 'dp_id','dp_port', 'vs_id', 'vs_port')) + for entry in entries: + self.app.stdout.write("{:<18} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18} {:<8}\n".\ + format(format_id(entry['vm_id']), entry['vm_port'], entry['eth_addr'],\ + entry['ct_id'], entry['dp_id'], entry['dp_port'], entry['vs_id'], entry['vs_port'])) + + self.app.stdout.write("\n") + + def view_isltable(self, **kwargs): + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rfisltable()) + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18}\n\n".\ + format('vm_id', 'ct_id', 'dp_id', 'dp_port',\ + 'eth_addr', 'rem_ct','rem_dp', 'rm_port', 'rem_addr')) + for entry in entries: + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18}\n".\ + format(format_id(entry['vm_id']), entry['ct_id'], entry['dp_id'], entry['dp_port'],\ + entry['eth_addr'], entry['rem_ct'], entry['rem_id'], entry['rem_port'], entry['rem_eth_addr'])) + + self.app.stdout.write("\n") + + def view_islconfig(self, **kwargs): + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rfislconfig()) + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18}\n\n".\ + format('vm_id', 'ct_id', 'dp_id', 'dp_port',\ + 'eth_addr', 'rem_ct','rem_dp', 'rm_port', 'rem_addr')) + for entry in entries: + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18}\n".\ + format(format_id(entry['vm_id']), entry['ct_id'], entry['dp_id'], entry['dp_port'],\ + entry['eth_addr'], entry['rem_ct'], entry['rem_id'], entry['rem_port'], entry['rem_eth_addr'])) + + self.app.stdout.write("\n") + + def view_config(self, **kwargs): + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rfconfig()) + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n\n".\ + format('vm_id', 'vm_port', 'ct_id', 'dp_id', 'dp_port')) + for entry in entries: + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n".\ + format(format_id(entry['vm_id']), entry['vm_port'], entry['ct_id'], entry['dp_id'], entry['dp_port'])) + + self.app.stdout.write("\n") + + def view_vmports(self, **kwargs): + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rfvmports()) + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n\n".\ + format('vm_id', 'vm_port', 'vs_id', 'vs_port','eth_addr')) + for entry in entries: + self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n".\ + format(format_id(entry['vm_id']), entry['vm_port'], entry['vs_id'], entry['vs_port'], entry['eth_addr'])) + + self.app.stdout.write("\n") + + def view_dpports(self, **kwargs): + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rfdpports()) + self.app.stdout.write("{:<8} {:<8} {:<8}\n\n".\ + format('ct_id', 'dp_port', 'dp_port')) + for entry in entries: + self.app.stdout.write("{:<8} {:<8} {:<8}\n".\ + format(entry['ct_id'], entry['dp_id'], entry['dp_port'])) + + self.app.stdout.write("\n") + +class RFServerCLI(App): + log = logging.getLogger(__name__) + log.setLevel(logging.INFO) + def __init__(self): + self.rfserver = xmlrpclib.ServerProxy('http://localhost:8008') + command = CommandManager('RFServer') + super(RFServerCLI, self).__init__( + description = 'RFServer', + version = '0.1', + command_manager = command,) + commands = { + 'delete': DeleteCommand, + 'update': UpdateCommand, + 'view': ViewCommand,} + for k,v in commands.iteritems(): + command.add_command(k, v) + + def initialize_app(self, argv): + self.log.debug('initialize_app') + def prepare_to_run_command(self, cmd): + self.log.debug('prepare_to_run_command %s', cmd.__class__.__name__) + + def clean_up(self, cmd, result, err): + self.log.debug('clean_up %s', cmd.__class__.__name__) + if err: + self.log.debug('got an error: %s', err) + + +def main(argv=sys.argv[1:]): + myapp = RFServerCLI() + return myapp.run(argv) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/rfserver/rfserverrpc.py b/rfserver/rfserverrpc.py new file mode 100755 index 00000000..b51dea4e --- /dev/null +++ b/rfserver/rfserverrpc.py @@ -0,0 +1,58 @@ +import logging + +import json +from SimpleXMLRPCServer import SimpleXMLRPCServer + +class RFServerRPC(): + def __init__(self, rfserver): + self.rfserver = rfserver + self.rpcserver = SimpleXMLRPCServer(("localhost", 8008)) + + self.rpcserver.register_instance(RPC_processor(self.rfserver)) + print "RPC server started" + self.rpcserver.serve_forever() + +class RPC_processor(): + def __init__(self, rfserver): + self.rfserver = rfserver + + def get_rftable(self): + entries = self.rfserver.rftable.get_entries() + results = [] + for entry in entries: + results.append(entry.to_dict()) + + return json.dumps(results) + + def get_rfconfig(self): + entries = self.rfserver.config.get_entries() + results = [] + for entry in entries: + results.append(entry.to_dict()) + + return json.dumps(results) + + def get_rfvmports(self): + entries = self.rfserver.vmporttable.get_entries() + results = [] + for entry in entries: + results.append(entry.to_dict()) + + return json.dumps(results) + + def get_rfdpports(self): + entries = self.rfserver.dpporttable.get_entries() + results = [] + for entry in entries: + results.append(entry.to_dict()) + + return json.dumps(results) + + def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, dp_port=None): + return self.rfserver.delete_single_mapping(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + #return True + + def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, dp_port=None): + return self.rfserver.update_single_mapping(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) \ No newline at end of file diff --git a/rfserver/rftable.py b/rfserver/rftable.py index 71710ec3..08f654a4 100644 --- a/rfserver/rftable.py +++ b/rfserver/rftable.py @@ -18,6 +18,9 @@ RFISLCONFENTRY = 2 RFISLENTRY = 3 +RFVMPORTENTRY = 4 +RFDPPORTENTRY = 5 + class EntryFactory: @staticmethod def make(type_): @@ -29,6 +32,10 @@ def make(type_): return RFISLEntry() elif type_ == RFISLCONFENTRY: return RFISLConfEntry() + elif type_ == RFVMPORTENTRY: + return RFVMPortEntry() + elif type_ == RFDPPORTENTRY: + return RFDPPortEntry(); class EntryTable(TableBase): def __init__(self, name, entry_type): @@ -195,7 +202,21 @@ def _is_idle_vm_port(self): self.dp_port is None and self.vs_id is None and self.vs_port is None) - + + # Used for dynamic mapping to update dp port info of an entry + def update_dp_port(self, ct_id = None,dp_id=None, dp_port=None): + self.ct_id = ct_id + self.dp_id = dp_id + self.dp_port = dp_port + + # Used for dynamic mapping to update vm port info of an entry + def update_vm_port(self, vm_id = None, vm_port=None, vs_id=None, vs_port=None, eth_addr=None): + self.vm_id = vm_id + self.vm_port = vm_port + self.vs_id = vs_id + self.vs_port = vs_port + self.eth_addr = eth_addr + def _is_idle_dp_port(self): return (self.vm_id is None and self.vm_port is None and @@ -450,7 +471,18 @@ def __init__(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, self.ct_id = ct_id self.dp_id = dp_id self.dp_port = dp_port - + + # Used for dynamic mapping to update dp port info of an entry + def update_dp_port(self, ct_id, dp_id, dp_port): + self.ct_id = ct_id + self.dp_id = dp_id + self.dp_port = dp_port + + # Used for dynamic mapping to update vm port info of an entry + def update_vm_port(self, vm_id, vm_port): + self.vm_id = vm_id + self.vm_port = vm_port + def __str__(self): return "vm_id: %s vm_port: %s "\ "dp_id: %s dp_port: %s "\ @@ -476,3 +508,85 @@ def to_dict(self): pack_into_dict(data, self, "dp_id") pack_into_dict(data, self, "dp_port") return data + +class RFVMPortEntry: + def __init__(self, id=None, vm_id=None, vm_port=None, eth_addr=None, vs_id=None, vs_port=None, state=None): + self.id = id + self.vm_id = vm_id + self.vm_port = vm_port + self.eth_addr = eth_addr + self.vs_id = vs_id + self.vs_port = vs_port + self.state = state + + def update_vs(self, vs_id, vs_port): + self.vs_id = vs_id + self.vs_port = vs_port + + def update_eth_addr(self, eth_addr): + self.eth_addr = eth_add + + def to_dict(self): + data = {} + if self.id is not None: + data["_id"] = self.id + pack_into_dict(data, self, "vm_id") + pack_into_dict(data, self, "vm_port") + pack_into_dict(data, self, "vs_id") + pack_into_dict(data, self, "vs_port") + pack_into_dict(data, self, "eth_addr") + return data + + def from_dict(self, data): + self.id = data["_id"] + load_from_dict(data, self, "vm_id") + load_from_dict(data, self, "vm_port") + load_from_dict(data, self, "eth_addr") + load_from_dict(data, self, "vs_id") + load_from_dict(data, self, "vs_port") + +class RFVMPortTable(EntryTable): + def __init__(self): + EntryTable.__init__(self, RFVMPORTTABLE_NAME, RFVMPORTENTRY) + + def get_vm_port_info(self, vm_id, vm_port): + result = self.get_entries(vm_id=vm_id, + vm_port=vm_port) + if not result: + return None + return result[0] + + +class RFDPPortTable(EntryTable): + def __init__(self): + EntryTable.__init__(self, RFDPPORTTABLE_NAME, RFDPPORTENTRY) + + def get_dp_port_info(self, ct_id, dp_id, dp_port): + result = self.get_entries(ct_id= ct_id, dp_id=dp_id, + dp_port=dp_port) + if not result: + return None + return result[0] + +class RFDPPortEntry: + def __init__(self, id=None, ct_id=None, dp_id=None, dp_port=None, state=None): + self.id = id + self.ct_id = ct_id + self.dp_id = dp_id + self.dp_port = dp_port + self.state = state + + def to_dict(self): + data = {} + if self.id is not None: + data["_id"] = self.id + pack_into_dict(data, self, "ct_id") + pack_into_dict(data, self, "dp_port") + pack_into_dict(data, self, "dp_id") + return data + + def from_dict(self, data): + self.id = data["_id"] + load_from_dict(data, self, "ct_id") + load_from_dict(data, self, "dp_port") + load_from_dict(data, self, "dp_id") \ No newline at end of file From 53a3c0679b8debf832ae87ca7efce6f87f4809b4 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 7 Jan 2015 15:50:53 +1300 Subject: [PATCH 16/52] Update projectw.sh for tests Update RPC server and client upload test script for Mininet (mytest/test2.py) Signed-off-by: Trung Truong --- mytest/test2.py | 68 +++++++++++++++ projectw.sh | 178 ++++++++++++++++++++++------------------ rfserver/rfservercli.py | 15 ++-- rfserver/rfserverrpc.py | 17 ++-- 4 files changed, 187 insertions(+), 91 deletions(-) create mode 100755 mytest/test2.py diff --git a/mytest/test2.py b/mytest/test2.py new file mode 100755 index 00000000..912f5daa --- /dev/null +++ b/mytest/test2.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +import sys +from mininet.net import Mininet +from mininet.node import Controller, RemoteController, OVSController +from mininet.node import CPULimitedHost, Host, Node +from mininet.node import OVSKernelSwitch, UserSwitch +from mininet.node import IVSSwitch +from mininet.cli import CLI +from mininet.log import setLogLevel, info +from mininet.link import TCLink, Intf + +# +# This will create two switches with one inter-link +# It takes input argument as the number of host ports for each sw +# the topology will look like +# h[1-N] ----(2-N)sw1(1)--------(1)sw2(2-N)------- h[N-(2*N-1)] +# + +def myNetwork(num_hosts): + net = Mininet(topo=None, build=False, ipBase='172.16.0.0/16') + info('*** Add controller\n') + c0 = net.addController(name='c0', controller=RemoteController, ip='192.168.56.101', port=6633) + + info('*** Add switches\n') + s1 = net.addSwitch(name='s1', cls=OVSKernelSwitch) + s2 = net.addSwitch(name='s2', cls=OVSKernelSwitch) + net.addLink(s1, s2) + + info('*** Add hosts\n') + hosts = [] + for i in range(1, 2*num_hosts + 1): + hostname = 'h%d' % i + addr = '172.16.%d.1/24' % (i%254) + gw = "via 172.16.%d.254" % (i%254) + host = net.addHost(name=hostname, cls=Host, ip=addr, defaultRoute=gw) + hosts.append(host) + if (i <= num_hosts): + net.addLink(hosts[i-1], s1) + else: + net.addLink(hosts[i-1], s2) + + info('Start network\n') + net.build() + for controller in net.controllers: + controller.start() + + info('Start switches\n') + net.get('s1').start([c0]) + net.get('s2').start([c0]) + + s1.cmd('ovs-vsctl set bridge s1 protocols=OpenFlow13') + s2.cmd('ovs-vsctl set bridge s2 protocols=OpenFlow13') + + info('Configure switches\n') + CLI(net) + net.stop() + +if __name__ == '__main__': + setLogLevel('info') + if len(sys.argv) == 0: + num_hosts = 1 # Number of host per switch + else: + try: + num_hosts = int(sys.argv[1]) + except: + num_hosts = 1 + + myNetwork(num_hosts) diff --git a/projectw.sh b/projectw.sh index 4a2365a7..476f4b14 100755 --- a/projectw.sh +++ b/projectw.sh @@ -1,25 +1,26 @@ #!/bin/bash # Set to 0 to use external switch(es) -STARTBVMS=1 +STARTBVMS=0 # If rfserverconfig.csv and rfserverinternal.csv exist in /home/projectw, # use them and don't try to configure rfvm1. If they do not exist, use values # below for default config. -DPPORTNET=172.31 +DPPORTNET=172.16 DPPORTNETV6=fc00:: -DPPORTS=2 -SWITCH1DPID=0x99 +DPPORTS=$2 # Number of ports per datapath +SWITCH1DPID=0x01 +SWITCH2DPID=0x02 MULTITABLEDPS="''" SATELLITEDPS="''" -HOME=/home/projectw +HOME=/home/ubuntu RF_HOME=$HOME/RouteFlow RFSERVERCONFIG=/tmp/rfserverconfig.csv RFSERVERINTERNAL=/tmp/rfserverinternal.csv HOME_RFSERVERCONFIG="$HOME/"`basename $RFSERVERCONFIG` HOME_RFSERVERINTERNAL="$HOME/"`basename $RFSERVERINTERNAL` -CONTROLLER_PORT=6653 +CONTROLLER_PORT=6633 LXCDIR=/var/lib/lxc RFVM1=$LXCDIR/rfvm1 RFBR=br0 @@ -28,8 +29,10 @@ RFDPID=7266767372667673 OFP=OpenFlow13 RFVM1IP=192.168.10.100 HOSTVMIP=192.168.10.1 -OVSSOCK=/tmp/openvswitch-db.sock -VSCTL="ovs-vsctl --db=unix:$OVSSOCK" +#OVSSOCK=/tmp/openvswitch-db.sock +#VSCTL="ovs-vsctl --db=unix:$OVSSOCK" +#OFCTL="ovs-ofctl -O$OFP" +VSCTL="ovs-vsctl" OFCTL="ovs-ofctl -O$OFP" export PATH=$PATH:/usr/local/bin:/usr/local/sbin export PYTHONPATH=$PYTHONPATH:$RF_HOME @@ -145,78 +148,93 @@ default_config() { # Default rfserver config cp /dev/null $RFSERVERINTERNAL echo "vm_id,ct_id,dp_id,dp_port,eth_addr,rem_ct,rem_id,rem_port,rem_eth_addr" > $RFSERVERINTERNAL + for i in `seq 1 $((2*$DPPORTS))`; do + for j in `seq $(($i + 1)) $((2*$DPPORTS))`; do + cat >> $RFSERVERINTERNAL < $RFSERVERCONFIG - for i in `seq 1 $DPPORTS` ; do - echo 0x12a0a0a0a0a0,$i,0,$SWITCH1DPID,$i >> $RFSERVERCONFIG + for i in `seq 1 $((2*$DPPORTS))` ; do + if [ $i -le $DPPORTS ]; then + echo 0x12a0a0a0a0a0,$i,0,$SWITCH1DPID,$(($i+1)) >> $RFSERVERCONFIG + else + echo 0x12a0a0a0a0a0,$i,0,$SWITCH2DPID,$(($i-$DPPORTS + 1)) >> $RFSERVERCONFIG + fi done # Configure the VM cat > $RFVM1/config <> $RFVM1/config<> $RFVM1/config< $ROOTFS/etc/network/interfaces < $ROOTFS/etc/network/interfaces <> $RFVM1/rootfs/etc/network/interfaces<> $RFVM1/rootfs/etc/network/interfaces< $ROOTFS/etc/rc.local < $ROOTFS/root/run_rfclient.sh < /var/log/rfclient.log 2> /var/log/rfclient.log.err + #!/bin/sh + /opt/rfclient/rfclient > /var/log/rfclient.log 2> /var/log/rfclient.log.err EOF + chmod +x $RFVM1/rootfs/root/run_rfclient.sh # Create the rfclient dir @@ -241,6 +260,7 @@ EOF mkdir -p $RFCLIENTDIR # Copy the rfclient executable + cd $RF_HOME cp rfclient/rfclient $RFCLIENTDIR/rfclient cp -p -P /usr/local/lib/libzmq* $ROOTFS/usr/local/lib chroot $ROOTFS ldconfig @@ -253,13 +273,13 @@ EOF reset() { echo_bold "-> Stopping and resetting LXC VMs..."; lxc-stop -n rfvm1 &> /dev/null; - lxc-stop -n b1 &> /dev/null; - lxc-stop -n b2 &> /dev/null; + #lxc-stop -n b1 &> /dev/null; + #lxc-stop -n b2 &> /dev/null; init=$1; if [ $init -eq 1 ]; then echo_bold "-> Starting OVS daemons..."; - start_ovs + #start_ovs else echo_bold "-> Stopping child processes..."; kill_process_tree 1 $$ @@ -298,11 +318,13 @@ if [ "$ACTION" != "RESET" ]; then ifconfig $RFBR $HOSTVMIP echo_bold "-> Starting RFServer..." - nice ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + #winpdb ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & echo_bold "-> Starting the controller ($ACTION) and RFPRoxy..." case "$ACTION" in RYU) + cd .. cd ryu-rfproxy ryu-manager --use-stderr --ofp-tcp-listen-port=$CONTROLLER_PORT ryu-rfproxy/rfproxy.py & ;; diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py index 2f89209f..2b9fdf72 100755 --- a/rfserver/rfservercli.py +++ b/rfserver/rfservercli.py @@ -33,16 +33,15 @@ def get_parser(self, prog_name): return parser def take_action(self, parsed_args): - vm_id = None if parsed_args.vm_id is None else int(str(parsed_args.vm_id), 16) + vm_id = None if parsed_args.vm_id is None else str(parsed_args.vm_id) vm_port = None if parsed_args.vm_port is None else int(str(parsed_args.vm_port)) - dp_id = None if parsed_args.dp_id is None else int(str(parsed_args.dp_id), 16) + dp_id = None if parsed_args.dp_id is None else str(parsed_args.dp_id) dp_port = None if parsed_args.dp_port is None else int(str(parsed_args.dp_port)) ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) rfserver = self.app.rfserver - result = rfserver.delete_single_mapping(vm_id=vm_id, vm_port=vm_port, - ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + result = rfserver.delete_single_mapping(vm_id, vm_port, ct_id, dp_id, dp_port) if result: self.app.log.info("Delete operation successful") else: @@ -62,15 +61,15 @@ def get_parser(self, prog_name): return parser def take_action(self, parsed_args): - vm_id = None if parsed_args.vm_id is None else int(str(parsed_args.vm_id), 16) + vm_id = None if parsed_args.vm_id is None else str(parsed_args.vm_id) vm_port = None if parsed_args.vm_port is None else int(str(parsed_args.vm_port)) - dp_id = None if parsed_args.dp_id is None else int(str(parsed_args.dp_id), 16) + dp_id = None if parsed_args.dp_id is None else str(parsed_args.dp_id) dp_port = None if parsed_args.dp_port is None else int(str(parsed_args.dp_port)) ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) rfserver = self.app.rfserver - result = rfserver.update_single_mapping(vm_id=vm_id, vm_port=vm_port, ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + result = rfserver.update_single_mapping(vm_id, vm_port, ct_id, dp_id, dp_port) if result: self.app.log.info("Delete operation successful") else: @@ -180,7 +179,7 @@ class RFServerCLI(App): log = logging.getLogger(__name__) log.setLevel(logging.INFO) def __init__(self): - self.rfserver = xmlrpclib.ServerProxy('http://localhost:8008') + self.rfserver = xmlrpclib.ServerProxy('http://localhost:8008', allow_none=True) command = CommandManager('RFServer') super(RFServerCLI, self).__init__( description = 'RFServer', diff --git a/rfserver/rfserverrpc.py b/rfserver/rfserverrpc.py index b51dea4e..993e02f6 100755 --- a/rfserver/rfserverrpc.py +++ b/rfserver/rfserverrpc.py @@ -6,7 +6,7 @@ class RFServerRPC(): def __init__(self, rfserver): self.rfserver = rfserver - self.rpcserver = SimpleXMLRPCServer(("localhost", 8008)) + self.rpcserver = SimpleXMLRPCServer(("localhost", 8008), allow_none=True) self.rpcserver.register_instance(RPC_processor(self.rfserver)) print "RPC server started" @@ -48,11 +48,18 @@ def get_rfdpports(self): return json.dumps(results) - def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, dp_port=None): + def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + if vm_id is not None: + vm_id = int(vm_id, 16) + if dp_id is not None: + dp_id = int(dp_id, 16) return self.rfserver.delete_single_mapping(vm_id=vm_id, vm_port=vm_port, ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - #return True - def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, dp_port=None): + def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + if vm_id is not None: + vm_id = int(vm_id, 16) + if dp_id is not None: + dp_id = int(dp_id, 16) return self.rfserver.update_single_mapping(vm_id=vm_id, vm_port=vm_port, - ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) \ No newline at end of file + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) \ No newline at end of file From e5b44a782772b735f7330102ef12a27f3731c424 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 7 Jan 2015 21:21:09 +1300 Subject: [PATCH 17/52] Changes to work with removing flow entries by output matching Signed-off-by: Trung Truong --- rflib/ipc/RFProtocol.py | 16 +++++++++++++++- rflib/types/Match.py | 2 +- rfserver/rfserver.py | 12 ++++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/rflib/ipc/RFProtocol.py b/rflib/ipc/RFProtocol.py index 662b6853..610eb878 100644 --- a/rflib/ipc/RFProtocol.py +++ b/rflib/ipc/RFProtocol.py @@ -408,7 +408,7 @@ def __str__(self): class RouteMod(IPCMessage): def __init__(self, mod=None, id=None, vm_port=None, table=None, group=None, - matches=None, actions=None, options=None): + matches=None, actions=None, options=None, outport=None): self.set_mod(mod) self.set_id(id) self.set_vm_port(vm_port) @@ -417,6 +417,14 @@ def __init__(self, mod=None, id=None, vm_port=None, table=None, group=None, self.set_matches(matches) self.set_actions(actions) self.set_options(options) + self.set_outport(outport) + + # This is for dynamic mapping, to delete entries based on output + def get_outport(self): + return self.outport + + def set_outport(self, outport): + self.outport = outport def get_type(self): return ROUTE_MOD @@ -519,6 +527,11 @@ def from_dict(self, data): self.set_matches(data["matches"]) self.set_actions(data["actions"]) self.set_options(data["options"]) + try: + outport = data["outport"] + except: + outport = None + self.set_outport(outport) def to_dict(self): data = {} @@ -530,6 +543,7 @@ def to_dict(self): data["matches"] = self.get_matches() data["actions"] = self.get_actions() data["options"] = self.get_options() + data["outport"] = self.get_outport() return data def __str__(self): diff --git a/rflib/types/Match.py b/rflib/types/Match.py index 99182be8..93ea5291 100644 --- a/rflib/types/Match.py +++ b/rflib/types/Match.py @@ -51,7 +51,7 @@ def MPLS(cls, label): @classmethod def IN_PORT(cls, port): return cls(RFMT_IN_PORT, port) - + @classmethod def VLAN_ID(cls, tag): return cls(RFMT_VLAN_ID, tag) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index fd1fdece..59f19134 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -134,14 +134,18 @@ def handle_isl_route_mod(self, r, rm): return rms # This method used to remove flow entries from switches when # users make a change (i.e. delete) of an association - def dp_flows_remove_by_vm_port(self, entry): + def dp_flows_table_update(self, entry): rms = [] + rm = RouteMod(RMT_DELETE, self.dp_id) rm.add_match(Match.IN_PORT(entry.dp_port)) - rm.add_match(Match.ETHERNET(entry.eth_addr)) rm.add_option(self.DEFAULT_PRIORITY) rms.append(rm) - print rm + + rm = RouteMod(RMT_DELETE, self.dp_id) + rm.set_outport(entry.dp_port) + rm.add_option(self.DEFAULT_PRIORITY) + rms.append(rm) return rms class SatelliteRouteModTranslator(DefaultRouteModTranslator): @@ -554,7 +558,7 @@ def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, d (format_id(rf_entry.dp_id), rf_entry.dp_port, format_id(rf_entry.vs_id), rf_entry.vs_port)) #TODO: Update flow table of related dp translator = self.route_mod_translator[rf_entry.dp_id] - rms = translator.dp_flows_remove_by_vm_port(rf_entry) + rms = translator.dp_flows_table_update(rf_entry) for rm in rms: self.send_route_mod(rf_entry.ct_id, rm) return True From 4baf28ae89685a3d1759eed30e9e9ba50f350d03 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Thu, 8 Jan 2015 00:27:34 +1300 Subject: [PATCH 18/52] One shell script and a Mininet script for dynamic testing Signed-off-by: Trung Truong --- dynamic-test.sh | 240 +++++++++++++++++++++++++++++++++++++++++ mytest/dynamic-test.py | 67 ++++++++++++ 2 files changed, 307 insertions(+) create mode 100755 dynamic-test.sh create mode 100755 mytest/dynamic-test.py diff --git a/dynamic-test.sh b/dynamic-test.sh new file mode 100755 index 00000000..fc0b2f4c --- /dev/null +++ b/dynamic-test.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +# Set to 0 to use external switch(es) +STARTBVMS=0 + +MULTITABLEDPS="''" +SATELLITEDPS="''" + +HOME=/home/ubuntu +RF_HOME=$HOME/RouteFlow +RFSERVERCONFIG=/tmp/rfserverconfig.csv +RFSERVERINTERNAL=/tmp/rfserverinternal.csv +HOME_RFSERVERCONFIG="$HOME/"`basename $RFSERVERCONFIG` +HOME_RFSERVERINTERNAL="$HOME/"`basename $RFSERVERINTERNAL` +CONTROLLER_PORT=6633 +LXCDIR=/var/lib/lxc +RFBR=br0 +RFDP=dp0 +RFDPID=7266767372667673 +OFP=OpenFlow13 +HOSTVMIP=192.168.10.1 +VSCTL="ovs-vsctl" +OFCTL="ovs-ofctl -O$OFP" +export PATH=$PATH:/usr/local/bin:/usr/local/sbin +export PYTHONPATH=$PYTHONPATH:$RF_HOME + +#modprobe 8021q +ulimit -c 1000000000 + +if [ "$EUID" != "0" ]; then + echo "You must be root to run this script." + exit 1 +fi + +ACTION="" +case "$1" in +--ryu) + ACTION="RYU" + ;; +--reset) + ACTION="RESET" + ;; +*) + echo "Invalid argument: $1" + echo "Options: " + echo " --ryu: run using RYU" + echo " --reset: stop running and clear data from previous executions" + exit + ;; +esac + +cd $RF_HOME + +wait_port_listen() { + port=$1 + while ! `nc -z localhost $port` ; do + echo -n . + sleep 1 + done +} + +echo_bold() { + echo -e "\033[1m${1}\033[0m" +} + +kill_process_tree() { + top=$1 + pid=$2 + + children=`ps -o pid --no-headers --ppid ${pid}` + + for child in $children + do + kill_process_tree 0 $child + done + + if [ $top -eq 0 ]; then + kill -9 $pid &> /dev/null + fi +} + +add_local_br() { + br=$1 + dpid=$2 + $VSCTL add-br $br + $VSCTL set bridge $br protocols=$OFP + if [ "$dpid" != "" ] ; then + $VSCTL set bridge $br other-config:datapath-id=$dpid + fi + ifconfig $br up + check_local_br_up $br +} + +check_local_br_up() { + br=$1 + echo waiting for OVS sw/controller $br to come up + while ! $OFCTL ping $br 64|grep -q "64 bytes from" ; do + echo -n "." + sleep 1 + done +} + +start_ovs() { + if [ ! -f /usr/local/etc/openvswitch/conf.db ] ; then + ovsdb-tool create /usr/local/etc/openvswitch/conf.db /usr/local/share/openvswitch/vswitch.ovsschema + fi + ovsdb-server --pidfile --detach --remote=punix:$OVSSOCK + ovs-vswitchd --pidfile --detach unix:$OVSSOCK +} + +start_rfvms() { + for vm in rfvmA rfvmB rfvmC rfvmD rfvmE; do + ROOTFS=$LXCDIR/$vm/rootfs + cp /dev/null $ROOTFS/var/log/syslog + VMLOG=/tmp/$vm.log + rm -f $VMLOG + lxc-start -n $vm -l DEBUG -o $VMLOG -d + done +} + +stop_rfvms() { + echo_bold "-> Stopping the virtual machines..." + for vm in rfvmA rfvmB rfvmC rfvmD rfvmE; do + lxc-stop -n $vm &> /dev/null; + ROOTFS=$LXCDIR/$vm/rootfs + rm -rf $ROOTFS/var/run/network/ifstate; + done +} + +reset() { + echo_bold "-> Stopping and resetting LXC VMs..."; + stop_rfvms + + init=$1; + if [ $init -eq 1 ]; then + echo_bold "-> Starting OVS daemons..."; + #start_ovs + else + echo_bold "-> Stopping child processes..."; + kill_process_tree 1 $$ + fi + + sudo $VSCTL del-br $RFBR &> /dev/null; + sudo $VSCTL del-br $RFDP &> /dev/null; + sudo $VSCTL emer-reset &> /dev/null; +} +reset 1 +trap "reset 0; exit 0" INT + +if [ "$ACTION" != "RESET" ]; then + if [ -f "$HOME_RFSERVERCONFIG" ] && [ -f "$HOME_RFSERVERINTERNAL" ] ; then + echo_bold "-> Using existing external config..." + cp $HOME_RFSERVERCONFIG $RFSERVERCONFIG + cp $HOME_RFSERVERINTERNAL $RFSERVERINTERNAL + else + echo_bold "-> Run with default config..." + cp /dev/null > $RFSERVERCONFIG + echo "vm_id,vm_port,ct_id,dp_id,dp_port" > $RFSERVERCONFIG + + echo 0x2a0a0a0a0a0,1,0,0x01,1 >> $RFSERVERCONFIG + echo 0x2a0a0a0a0a0,2,0,0x01,2 >> $RFSERVERCONFIG + + echo 0x2b0b0b0b0b0,1,0,0x02,1 >> $RFSERVERCONFIG + echo 0x2b0b0b0b0b0,2,0,0x02,2 >> $RFSERVERCONFIG + + echo 0x2c0c0c0c0c0,3,0,0x01,3 >> $RFSERVERCONFIG + + echo 0x2d0d0d0d0d0,3,0,0x02,3 >> $RFSERVERCONFIG + + echo 0x2e0e0e0e0e0,1,0,0x03,1 >> $RFSERVERCONFIG + echo 0x2e0e0e0e0e0,2,0,0x03,2 >> $RFSERVERCONFIG + + cp /dev/null $RFSERVERINTERNAL + echo "vm_id,ct_id,dp_id,dp_port,eth_addr,rem_ct,rem_id,rem_port,rem_eth_addr" > $RFSERVERINTERNAL + fi + + echo_bold "-> Starting the management network ($RFBR)..." + add_local_br $RFBR + ifconfig $RFBR $HOSTVMIP + + echo_bold "-> Starting RFServer..." + #winpdb ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + + echo_bold "-> Starting the controller ($ACTION) and RFPRoxy..." + case "$ACTION" in + RYU) + cd .. + cd ryu-rfproxy + ryu-manager --use-stderr --ofp-tcp-listen-port=$CONTROLLER_PORT ryu-rfproxy/rfproxy.py & + ;; + esac + cd - &> /dev/null + wait_port_listen $CONTROLLER_PORT + check_local_br_up tcp:127.0.0.1:$CONTROLLER_PORT + + echo_bold "-> Starting the control plane network ($RFDP VS)..." + $VSCTL add-br $RFDP + $VSCTL set bridge $RFDP other-config:datapath-id=$RFDPID + $VSCTL set bridge $RFDP protocols=$OFP + $VSCTL set-controller $RFDP tcp:127.0.0.1:$CONTROLLER_PORT + $OFCTL add-flow $RFDP actions=CONTROLLER:65509 + ifconfig $RFDP up + check_local_br_up $RFDP + + echo_bold "-> Waiting for $RFDP to connect to controller..." + while ! $VSCTL find Controller target=\"tcp:127.0.0.1:$CONTROLLER_PORT\" is_connected=true | grep -q connected ; do + echo -n . + sleep 1 + done + + echo_bold "-> Starting virtual machines..." + start_rfvms + while ! ifconfig -s rfvmA.0 ; do + echo -n . + sleep 1 + done + + # Add VM eth0 port to management bridge + for vm in rfvmA rfvmB rfvmC rfvmD rfvmE; do + $VSCTL add-port $RFBR $vm.0 + done + + # Add VM interfaces to dataplane bridge + for i in `netstat -i|grep rfvm|cut -f 1 -d " "` ; do + if [ "$i" != "`echo $i | grep .0`" ] ; then + $VSCTL add-port $RFDP $i + fi + done + + echo_bold "-> Waiting for VMs to come up..." + while ! ping -W 1 -c 1 192.168.10.101 ; do + echo -n . + sleep 1 + done + + echo_bold "You can stop this test by pressing Ctrl+C." + wait +fi +exit 0 diff --git a/mytest/dynamic-test.py b/mytest/dynamic-test.py new file mode 100755 index 00000000..402dd050 --- /dev/null +++ b/mytest/dynamic-test.py @@ -0,0 +1,67 @@ +#!/usr/bin/python +import sys +from mininet.net import Mininet +from mininet.node import Controller, RemoteController, OVSController +from mininet.node import CPULimitedHost, Host, Node +from mininet.node import OVSKernelSwitch, UserSwitch +from mininet.node import IVSSwitch +from mininet.cli import CLI +from mininet.log import setLogLevel, info +from mininet.link import TCLink, Intf + +# +# This will create two switches with one inter-link +# It takes input argument as the number of host ports for each sw +# the topology will look like +# h1 ----(1)sw1(2)--------(2)sw2(1)------- h2 +# (3) (3) +# \ / +# \ / +# \ / +# \ / +# \ / +# (1) (2) +# s3 + +def myNetwork(): + net = Mininet(topo=None, build=False, ipBase='172.16.0.0/16') + info('*** Add controller\n') + c0 = net.addController(name='c0', controller=RemoteController, ip='192.168.56.101', port=6633) + + info('*** Add switches\n') + s1 = net.addSwitch(name='s1', cls=OVSKernelSwitch) + s2 = net.addSwitch(name='s2', cls=OVSKernelSwitch) + s3 = net.addSwitch(name='s3', cls=OVSKernelSwitch) + + info('*** Add hosts\n') + h1 = net.addHost(name="h1", cls=Host, ip='172.16.1.1/24', defaultRoute='via 172.16.1.254') + h2 = net.addHost(name="h2", cls=Host, ip='172.16.2.1/24', defaultRoute='via 172.16.2.254') + + net.addLink(h1, s1) + net.addLink(h2, s2) + + net.addLink(s1, s2) + net.addLink(s1, s3) + net.addLink(s2, s3) + + info('Start network\n') + net.build() + for controller in net.controllers: + controller.start() + + info('Start switches\n') + net.get('s1').start([c0]) + net.get('s2').start([c0]) + net.get('s3').start([c0]) + + info('Configure switches\n') + s1.cmd('ovs-vsctl set bridge s1 protocols=OpenFlow13') + s2.cmd('ovs-vsctl set bridge s2 protocols=OpenFlow13') + s3.cmd('ovs-vsctl set bridge s3 protocols=OpenFlow13') + + CLI(net) + net.stop() + +if __name__ == '__main__': + setLogLevel('info') + myNetwork() \ No newline at end of file From d65806bc2f6788792f642a157ffa2cd460b6b484 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Thu, 8 Jan 2015 13:27:31 +1300 Subject: [PATCH 19/52] Version 2 of RF Dynamic, quite different from the last commit Here the things: To make it simple, this verion defines 3 methods: delete/add/update delete method removes entry from config table, rf table & notify proxy to update its table. It also removes flow entries based on in_port and out_port matches add method allows to add new config map between vm_port & dp_port, if they are not in use. update method allows to change the map config of a vm_port or a dp_port Signed-off-by: Trung Truong --- rflib/defs.py | 6 +- rfserver/rfserver.py | 261 +++++++++++++++++++++------------------- rfserver/rfservercli.py | 115 +++++++++++++----- rfserver/rfserverrpc.py | 34 +++++- 4 files changed, 252 insertions(+), 164 deletions(-) diff --git a/rflib/defs.py b/rflib/defs.py index a58eb845..b3125046 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -68,10 +68,8 @@ PC_RESET = 1 # Data plane Port Map Configuration (DCT_) -DCT_UPDATE_DP = 0 # Update Proxy map table by DP info -DCT_UPDATE_VS = 1 # Update Proxy map table by VS info -DCT_DELETE_DP = 2 # Delete Proxy map table by DP info -DCT_DELETE_VS = 3 # Delete Proxy map table by VS info +DCT_MAP_ADD = 0 # Add a Proxy map table entry +DCT_MAP_DELETE = 1 # Delete a Proxy map table entry # Format 12-digit hex ID format_id = lambda dp_id: hex(dp_id).rstrip("L") if (dp_id is not None) else 'None' diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 59f19134..504274c7 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -25,6 +25,7 @@ from rftable import * from rfserverrpc import RFServerRPC +from pycurl import CSELECT_ERR logging.basicConfig( level=logging.INFO, @@ -134,7 +135,7 @@ def handle_isl_route_mod(self, r, rm): return rms # This method used to remove flow entries from switches when # users make a change (i.e. delete) of an association - def dp_flows_table_update(self, entry): + def delete_flows_by_port(self, entry): rms = [] rm = RouteMod(RMT_DELETE, self.dp_id) @@ -531,153 +532,165 @@ def __init__(self, configfile, islconffile, multitabledps, satellitedps): self.ipc.listen(RFCLIENT_RFSERVER_CHANNEL, self, self, False) self.ipc.listen(RFSERVER_RFPROXY_CHANNEL, self, self, False) - # Delete a mapping configuration. - def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): - cf_entry = None - if vm_id is not None and vm_port is not None: - cf_entry = self.config.get_config_for_vm_port(vm_id, vm_port) - - elif dp_id is not None and dp_port is not None and ct_id is not None: - cf_entry = self.config.get_config_for_dp_port(ct_id, dp_id, dp_port) - - if cf_entry is not None: #Delete it from config table + # This method delete a mapping entry in Config table + # & an association entry in RFTable as well + # A PortConfig msg is sent to RFClient to update its port state. + # (May not be necessary). + # A RouteMod with DELETE command, matching in_port & output + # will be sent to RFProxy to update the flow table + # RFPRoxy need to use Barrier to guarantee the update + def delete_map_configs(self, **kwargs): + cf_entries = self.config.get_entries(**kwargs) + count = 0 + for cf_entry in cf_entries: + #TODO: Shouldn't remove entry at this stage. + # Should wait for proxy confirmation that the switch's table + # has been updated successfully self.config.remove_entry(cf_entry) - # Send reset signal to RFClient, so it starts generating PortMap msg - #self.reset_vm_port(cf_entry.vm_id, cf_entry.vm_port) - # Update map table in RFProxy - rf_entry = self.rftable.get_entry_by_vm_port(cf_entry.vm_id, cf_entry.vm_port) + + #Update the association table: RFTable + rf_entry = self.rftable.get_entry_by_vm_port(cf_entry.vm_id, + cf_entry.vm_port) + if rf_entry is None: + rf_entry = self.rftable.get_entry_by_dp_port(cf_entry.dp_id, + cf_entry.dp_port) if rf_entry is not None: - # Delete from RFTable & send a reset msg to RFProxy self.rftable.remove_entry(rf_entry) if rf_entry.get_status() == RFENTRY_ACTIVE: msg = DataPlaneMap(ct_id=rf_entry.ct_id, - dp_id=rf_entry.dp_id, dp_port=rf_entry.dp_port, - vs_id=rf_entry.vs_id, vs_port=rf_entry.vs_port, operation_id=DCT_DELETE_DP) - self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(rf_entry.ct_id), msg) - self.log.info("Resetting RFProxy map table (dp_id=%s, dp_port=%i) - (vs_id=%s, vs_port=%i)" % - (format_id(rf_entry.dp_id), rf_entry.dp_port, format_id(rf_entry.vs_id), rf_entry.vs_port)) - #TODO: Update flow table of related dp + dp_id=rf_entry.dp_id, + dp_port=rf_entry.dp_port, + vs_id=rf_entry.vs_id, + vs_port=rf_entry.vs_port, + operation_id=DCT_MAP_DELETE) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, + str(rf_entry.ct_id), msg) + self.log.info("Resetting RFProxy map table \ + (dp_id=%s, dp_port=%i) - (vs_id=%s, vs_port=%i)" % + (format_id(rf_entry.dp_id), + rf_entry.dp_port, format_id(rf_entry.vs_id), + rf_entry.vs_port)) + # Now update switch flow table translator = self.route_mod_translator[rf_entry.dp_id] - rms = translator.dp_flows_table_update(rf_entry) + rms = translator.delete_flows_by_port(rf_entry) for rm in rms: self.send_route_mod(rf_entry.ct_id, rm) - return True + + self.log.info("Successfully deleted a mapping (vm_id=%s, \ + vm_port=%i) - (dp_id=%s, dp_port=%i" % (format_id(cf_entry.vm_id), + cf_entry.vm_port, + format_id(cf_entry.dp_id), + cf_entry.dp_port)) + count += 1 + + return count - self.log.info("No config found") - return False + # This method creates a new mapping between a vm_port & a dp_port + def add_map_config(self, vm_id, vm_port, ct_id, dp_id, dp_port): + if vm_id is None or vm_port is None or \ + dp_id is None or dp_port is None or ct_id is None: + return False # Can't accept None - # Update or create a mapping. If not exist, create it. - # A VM port can be mapped to a new DP port if the DP port is not being used, similar to DP port mapping change. - def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): - if vm_id is None or vm_port is None \ - or dp_id is None or dp_port is None or ct_id is None: - return False + vm_port_conf = self.config.get_config_for_vm_port(vm_id=vm_id, + vm_port=vm_port) + dp_port_conf = self.config.get_config_for_dp_port(ct_id=ct_id, + dp_id=dp_id, + dp_port=dp_port) + if vm_port_conf is not None or dp_port_conf is not None: + return False # One or both are in used - vm_port_conf = self.config.get_config_for_vm_port(vm_id=vm_id, vm_port=vm_port) - dp_port_conf = self.config.get_config_for_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - old_conf = None - rf_entry = None - action = None - # 1. Update CONFIG table - if vm_port_conf is not None: - if dp_port_conf is not None: - # Both are in use, so do nothing - self.log.info("Both VM port and DP port are in used. No update can be done") - return False - else: # Map VM port to a new DP port - vm_port_conf.update_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - self.config.set_entry(vm_port_conf) - action = 1 - else: # Map DP port to a new VM port - if dp_port_conf is not None: # Map DP port to a new VM port - dp_port_conf.update_vm_port(vm_id=vm_id, vm_port=vm_port) - self.config.set_entry(dp_port_conf) - action = 2 - else: # Create a new mapping between VM port & DP port - self.config.set_entry(RFConfigEntry(vm_id=vm_id, vm_port=vm_port, - ct_id=ct_id, dp_id=dp_id, dp_port=dp_port)) - - action = 3 - - # 2. Update RFTable - rfclient_inform = False - if action == 1: # If update mapping of VM port - rf_entry = self.rftable.get_entry_by_vm_port(vm_id=vm_id, vm_port=vm_port) - if rf_entry is None: - rf_entry = RFEntry() - dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - if dp_port_info is not None: - rf_entry.update_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - if rf_entry.get_status() == RFENTRY_ACTIVE: - rfproxy_oper_id = DCT_UPDATE_VS - rfclient_inform = True - else: # We have no info about the new dp port - rf_entry.make_idle(RFENTRY_IDLE_VM_PORT) - # Reset rfproxy table - rfproxy_oper_id = DCT_DELETE_VS - # Update to RFTable - self.rftable.set_entry(rf_entry) - elif action == 2: # Update mapping of DP port - rf_entry = self.rftable.get_entry_by_dp_port(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - if rf_entry is None: - rf_entry = RFEntry() - vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) - if vm_port_info is not None: - rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, + cf_entry = RFConfigEntry(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + # Update the new entry to Config table + self.config.set_entry(cf_entry) + + # Now update RFTable + vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, + vm_port=vm_port) + dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, + dp_id=dp_id, + dp_port=dp_port) + if vm_port_info is None and dp_port_info is None: + return True # Successful in term of updating config + + rf_entry = RFEntry() + if vm_port_info is not None: + rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, vs_id=vm_port_info.vs_id, vs_port=vm_port_info.vs_port, eth_addr=vm_port_info.eth_addr) - self.rftable.set_entry(rf_entry) - if rf_entry.get_status() == RFENTRY_ACTIVE: - rfproxy_oper_id = DCT_UPDATE_DP - rfclient_inform = True - else: # We have no info about the new vm port - rf_entry.make_idle(RFENTRY_IDLE_DP_PORT) - self.rftable.set_entry(rf_entry) - rfproxy_oper_id = DCT_DELETE_DP - else: # Create a new mapping - rf_entry = RFEntry() - dp_port_info = self.dpporttable.get_dp_port_info(ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) - vm_port_info = self.vmporttable.get_vm_port_info(vm_id=vm_id, vm_port=vm_port) - - if dp_port_info is not None: - rf_entry.update_dp_port(dp_id=dp_id, dp_port=dp_port, ct_id=ct_id) - - if vm_port_info is not None: - rf_entry.update_vm_port(vm_id=vm_id, vm_port=vm_port, - vs_id=vm_port_info.vs_id, - vs_port=vm_port_info.vs_port, - eth_addr=vm_port_info.eth_addr) - if vm_port_info is None and dp_port_info is None: - rfproxy_oper_id = None - else: - self.rftable.set_entry(rf_entry) - if rf_entry.get_status() == RFENTRY_ACTIVE: - rfproxy_oper_id = DCT_UPDATE_DP - rfclient_inform = True + if dp_port_info is not None: + rf_entry.update_dp_port(dp_id=dp_id, dp_port=dp_port, ct_id=ct_id) + + self.rftable.set_entry(rf_entry) - if rfproxy_oper_id is not None: + if rf_entry.get_status() == RFENTRY_ACTIVE: msg = DataPlaneMap(ct_id=rf_entry.ct_id, dp_id=rf_entry.dp_id, dp_port=rf_entry.dp_port, vs_id=rf_entry.vs_id, vs_port=rf_entry.vs_port, - operation_id=rfproxy_oper_id) + operation_id=DCT_MAP_ADD) self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(rf_entry.ct_id), msg) - if rfclient_inform: - # Inform RFClient about the new mapping - msg = PortConfig(vm_id=vm_id, vm_port=vm_port, + msg = PortConfig(vm_id=rf_entry.vm_id, vm_port=rf_entry.vm_port, operation_id=PCT_MAP_SUCCESS) self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(rf_entry.vm_id), msg) + + self.log.info("Successfully updated RFTable \ + (dp_id=%s, dp_port=%i) - (vs_id=%s, vs_port=%i)" % + (format_id(rf_entry.dp_id), + rf_entry.dp_port, format_id(rf_entry.vs_id), + rf_entry.vs_port)) + + + return True + + # Update an existing mapping: + # change the mapping of a vm_port to another dp_port + # or change the mapping of dp_port to another vm_port + def update_map_config(self, vm_id, vm_port, ct_id, dp_id, dp_port): + + if vm_id is None or vm_port is None \ + or dp_id is None or dp_port is None or ct_id is None: + return False + + vm_port_conf = self.config.get_config_for_vm_port(vm_id=vm_id, + vm_port=vm_port) + + dp_port_conf = self.config.get_config_for_dp_port(ct_id=ct_id, + dp_id=dp_id, + dp_port=dp_port) + if vm_port_conf is not None and dp_port_conf is not None: + return False # Both are in used, can't update + + + if vm_port_conf is not None: # Map the vm_port to a new dp_port + # Delete the existing mapping first + if self.delete_map_configs(vm_id=vm_port_conf.vm_id, + vm_port=vm_port_conf.vm_port, + ct_id=vm_port_conf.ct_id, + dp_id=vm_port_conf.dp_id, + dp_port=vm_port_conf.dp_port): + + pass + else: + return False + + if dp_port_conf is not None: # Map the dp_port to a new vm_port + # Delete the existing mapping first + if self.delete_map_configs(vm_id=dp_port_conf.vm_id, + vm_port=dp_port_conf.vm_port, + ct_id=dp_port_conf.ct_id, + dp_id=dp_port_conf.dp_id, + dp_port=dp_port_conf.dp_port): + + pass + else: + return False - self.log.info("Updating a new client-datapath association " - "(vm_id=%s, vm_port=%s, dp_id=%s, " - "dp_port=%s, vs_id=%s, vs_port=%s)" % - (format_id(rf_entry.vm_id), rf_entry.vm_port, - format_id(rf_entry.dp_id), rf_entry.dp_port, - format_id(rf_entry.vs_id), rf_entry.vs_port)) + self.add_map_config(vm_id, vm_port, ct_id, dp_id, dp_port) return True def process(self, from_, to, channel, msg): @@ -854,7 +867,7 @@ def register_dp_port(self, ct_id, dp_id, dp_port): msg = DataPlaneMap(ct_id=entry.ct_id, dp_id=entry.dp_id, dp_port=entry.dp_port, - vs_id=entry.vs_id, vs_port=entry.vs_port, operation_id=DCT_UPDATE_DP) + vs_id=entry.vs_id, vs_port=entry.vs_port, operation_id=DCT_MAP_ADD) self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) msg = PortConfig(vm_id=entry.vm_id, vm_port=entry.vm_port, operation_id=PCT_MAP_SUCCESS) @@ -1003,7 +1016,7 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): self.rftable.set_entry(entry) msg = DataPlaneMap(ct_id=entry.ct_id, dp_id=entry.dp_id, dp_port=entry.dp_port, - vs_id=vs_id, vs_port=vs_port, operation_id=DCT_UPDATE_DP) + vs_id=vs_id, vs_port=vs_port, operation_id=DCT_MAP_ADD) self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) msg = PortConfig(vm_id=vm_id, vm_port=vm_port, operation_id=PCT_MAP_SUCCESS) diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py index 2b9fdf72..c6d31300 100755 --- a/rfserver/rfservercli.py +++ b/rfserver/rfservercli.py @@ -24,11 +24,16 @@ class DeleteCommand(Command): def get_parser(self, prog_name): parser = super(DeleteCommand, self).get_parser(prog_name) - parser.add_argument('-vm_id', '--vm_id', type=str, required=False, default=None) - parser.add_argument('-vm_port', '--vm_port', type=str, required=False, default=None) - parser.add_argument('-dp_id', '--dp_id', type=str, required=False, default=None) - parser.add_argument('-dp_port', '--dp_port', type=str, required=False, default=None) - parser.add_argument('-ct_id', '--ct_id', type=str, required=False, default=0) + parser.add_argument('-vm_id', '--vm_id', type=str, + required=False, default=None) + parser.add_argument('-vm_port', '--vm_port', type=str, + required=False, default=None) + parser.add_argument('-dp_id', '--dp_id', type=str, + required=False, default=None) + parser.add_argument('-dp_port', '--dp_port', type=str, + required=False, default=None) + parser.add_argument('-ct_id', '--ct_id', type=str, + required=False, default=0) return parser @@ -41,22 +46,58 @@ def take_action(self, parsed_args): ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) rfserver = self.app.rfserver - result = rfserver.delete_single_mapping(vm_id, vm_port, ct_id, dp_id, dp_port) - if result: - self.app.log.info("Delete operation successful") - else: - self.app.log.info("Delete operation failed") + count = \ + rfserver.delete_map_configs(vm_id, vm_port, ct_id, dp_id, dp_port) + + self.app.log.info("%i have been deleted" % count) + +class AddCommand(Command): + log = logging.getLogger(__name__) + + def get_parser(self, prog_name): + parser = super(AddCommand, self).get_parser(prog_name) + parser.add_argument('-vm_id', '--vm_id', type=str, + required=True, default=None) + parser.add_argument('-vm_port', '--vm_port', type=str, + required=True, default=None) + parser.add_argument('-dp_id', '--dp_id', type=str, + required=True, default=None) + parser.add_argument('-dp_port', '--dp_port', type=str, + required=True, default=None) + parser.add_argument('-ct_id', '--ct_id', type=str, + required=False, default=0) + return parser + + def take_action(self, parsed_args): + vm_id = None if parsed_args.vm_id is None else str(parsed_args.vm_id) + vm_port = None if parsed_args.vm_port is None else int(str(parsed_args.vm_port)) + + dp_id = None if parsed_args.dp_id is None else str(parsed_args.dp_id) + dp_port = None if parsed_args.dp_port is None else int(str(parsed_args.dp_port)) + ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) + + rfserver = self.app.rfserver + if rfserver.add_map_config(vm_id, vm_port, ct_id, dp_id, dp_port): + self.app.log.info("Done!") + else: + self.app.log.info("Failed!") + class UpdateCommand(Command): log = logging.getLogger(__name__) def get_parser(self, prog_name): parser = super(UpdateCommand, self).get_parser(prog_name) - parser.add_argument('-vm_id', '--vm_id', type=str, required=True, default=None) - parser.add_argument('-vm_port', '--vm_port', type=str, required=True, default=None) - parser.add_argument('-dp_id', '--dp_id', type=str, required=True, default=None) - parser.add_argument('-dp_port', '--dp_port', type=str, required=True, default=None) - parser.add_argument('-ct_id', '--ct_id', type=str, required=False, default=0) + parser.add_argument('-vm_id', '--vm_id', type=str, + required=True, default=None) + parser.add_argument('-vm_port', '--vm_port', type=str, + required=True, default=None) + parser.add_argument('-dp_id', '--dp_id', type=str, + required=True, default=None) + parser.add_argument('-dp_port', '--dp_port', type=str, + required=True, default=None) + parser.add_argument('-ct_id', '--ct_id', type=str, + required=None, default=0) return parser @@ -69,17 +110,17 @@ def take_action(self, parsed_args): ct_id = None if parsed_args.ct_id is None else int(str(parsed_args.ct_id)) rfserver = self.app.rfserver - result = rfserver.update_single_mapping(vm_id, vm_port, ct_id, dp_id, dp_port) - if result: - self.app.log.info("Delete operation successful") + if rfserver.update_map_config(vm_id, vm_port, ct_id, dp_id, dp_port): + self.app.log.info("Done!") else: - self.app.log.info("Delete operation failed") + self.app.log.info("Failed!") -class ViewCommand(Command): +class ViewCommand(Command): def get_parser(self, prog_name): parser = super(ViewCommand, self).get_parser(prog_name) parser.add_argument('view', nargs='?', default='rftable') - parser.add_argument('-filter', '--filter', type=str, required=False, default=None) + parser.add_argument('-filter', '--filter', type=str, + required=False, default=None) return parser @@ -111,8 +152,10 @@ def view_rftable(self, **kwargs): 'ct_id', 'dp_id','dp_port', 'vs_id', 'vs_port')) for entry in entries: self.app.stdout.write("{:<18} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18} {:<8}\n".\ - format(format_id(entry['vm_id']), entry['vm_port'], entry['eth_addr'],\ - entry['ct_id'], entry['dp_id'], entry['dp_port'], entry['vs_id'], entry['vs_port'])) + format(format_id(entry['vm_id']), entry['vm_port'], + entry['eth_addr'], entry['ct_id'], + entry['dp_id'], entry['dp_port'], + entry['vs_id'], entry['vs_port'])) self.app.stdout.write("\n") @@ -124,8 +167,11 @@ def view_isltable(self, **kwargs): 'eth_addr', 'rem_ct','rem_dp', 'rm_port', 'rem_addr')) for entry in entries: self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18}\n".\ - format(format_id(entry['vm_id']), entry['ct_id'], entry['dp_id'], entry['dp_port'],\ - entry['eth_addr'], entry['rem_ct'], entry['rem_id'], entry['rem_port'], entry['rem_eth_addr'])) + format(format_id(entry['vm_id']), + entry['ct_id'], entry['dp_id'], + entry['dp_port'], entry['eth_addr'], + entry['rem_ct'], entry['rem_id'], + entry['rem_port'], entry['rem_eth_addr'])) self.app.stdout.write("\n") @@ -137,8 +183,11 @@ def view_islconfig(self, **kwargs): 'eth_addr', 'rem_ct','rem_dp', 'rm_port', 'rem_addr')) for entry in entries: self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<18} {:<8} {:<8} {:<8} {:<18}\n".\ - format(format_id(entry['vm_id']), entry['ct_id'], entry['dp_id'], entry['dp_port'],\ - entry['eth_addr'], entry['rem_ct'], entry['rem_id'], entry['rem_port'], entry['rem_eth_addr'])) + format(format_id(entry['vm_id']), + entry['ct_id'], entry['dp_id'], + entry['dp_port'], entry['eth_addr'], + entry['rem_ct'], entry['rem_id'], + entry['rem_port'], entry['rem_eth_addr'])) self.app.stdout.write("\n") @@ -149,7 +198,9 @@ def view_config(self, **kwargs): format('vm_id', 'vm_port', 'ct_id', 'dp_id', 'dp_port')) for entry in entries: self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n".\ - format(format_id(entry['vm_id']), entry['vm_port'], entry['ct_id'], entry['dp_id'], entry['dp_port'])) + format(format_id(entry['vm_id']), + entry['vm_port'], entry['ct_id'], + entry['dp_id'], entry['dp_port'])) self.app.stdout.write("\n") @@ -160,7 +211,9 @@ def view_vmports(self, **kwargs): format('vm_id', 'vm_port', 'vs_id', 'vs_port','eth_addr')) for entry in entries: self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n".\ - format(format_id(entry['vm_id']), entry['vm_port'], entry['vs_id'], entry['vs_port'], entry['eth_addr'])) + format(format_id(entry['vm_id']), + entry['vm_port'], entry['vs_id'], + entry['vs_port'], entry['eth_addr'])) self.app.stdout.write("\n") @@ -171,7 +224,8 @@ def view_dpports(self, **kwargs): format('ct_id', 'dp_port', 'dp_port')) for entry in entries: self.app.stdout.write("{:<8} {:<8} {:<8}\n".\ - format(entry['ct_id'], entry['dp_id'], entry['dp_port'])) + format(entry['ct_id'], + entry['dp_id'], entry['dp_port'])) self.app.stdout.write("\n") @@ -188,6 +242,7 @@ def __init__(self): commands = { 'delete': DeleteCommand, 'update': UpdateCommand, + 'add' : AddCommand, 'view': ViewCommand,} for k,v in commands.iteritems(): command.add_command(k, v) diff --git a/rfserver/rfserverrpc.py b/rfserver/rfserverrpc.py index 993e02f6..525cf7c1 100755 --- a/rfserver/rfserverrpc.py +++ b/rfserver/rfserverrpc.py @@ -48,18 +48,40 @@ def get_rfdpports(self): return json.dumps(results) - def delete_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + def delete_map_configs(self, vm_id=None, vm_port=None, ct_id=None, dp_id=None, dp_port=None): + kwargs = {} + + if vm_id is not None: + kwargs['vm_id'] = int(vm_id, 16) + + if vm_port is not None: + kwargs['vm_port'] = vm_port + + if ct_id is not None: + kwargs['ct_id'] = ct_id + + if dp_id is not None: + kwargs['dp_id'] = int(dp_id, 16) + + if dp_port is not None: + kwargs['dp_port'] = dp_port + + return self.rfserver.delete_map_configs(**kwargs) + + def add_map_config(self, vm_id, vm_port, ct_id, dp_id, dp_port): if vm_id is not None: vm_id = int(vm_id, 16) if dp_id is not None: dp_id = int(dp_id, 16) - return self.rfserver.delete_single_mapping(vm_id=vm_id, vm_port=vm_port, - ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) + return self.rfserver.add_map_config(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, + dp_id=dp_id, dp_port=dp_port) - def update_single_mapping(self, vm_id=None, vm_port=None, ct_id=0, dp_id=None, dp_port=None): + def update_map_config(self, vm_id, vm_port, ct_id, dp_id, dp_port): if vm_id is not None: vm_id = int(vm_id, 16) if dp_id is not None: dp_id = int(dp_id, 16) - return self.rfserver.update_single_mapping(vm_id=vm_id, vm_port=vm_port, - ct_id=ct_id, dp_id=dp_id, dp_port=dp_port) \ No newline at end of file + return self.rfserver.update_map_config(vm_id=vm_id, vm_port=vm_port, + ct_id=ct_id, + dp_id=dp_id, dp_port=dp_port) \ No newline at end of file From a5893d985a7eeecb59058bdaca4c693ec5858119 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Fri, 9 Jan 2015 01:39:18 +1300 Subject: [PATCH 20/52] Update a few minor things Change in the way handling RouteMod with ISL: Search for specific VM. That's it, only install flows to ISL port that is configured for the vm Signed-off-by: Trung Truong --- rfserver/rfserver.py | 52 ++++++++++++++++++++++++++++++++++------- rfserver/rfservercli.py | 4 ++-- rfserver/rfserverrpc.py | 16 +++++++++++++ 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 504274c7..e8777f01 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -37,6 +37,8 @@ REGISTER_IDLE = 0 REGISTER_ASSOCIATED = 1 REGISTER_ISL = 2 +# Added to solve the problem when proxy dies and come to life again +REGISTER_ACTIVE = 3 class RouteModTranslator(object): @@ -113,7 +115,14 @@ def handle_route_mod(self, entry, rm): entries = self.rftable.get_entries(dp_id=entry.dp_id, ct_id=entry.ct_id) - entries.extend(self.isltable.get_entries(dp_id=entry.dp_id, + + #TODO: Restore this. This change used for testing only + #entries.extend(self.isltable.get_entries(dp_id=entry.dp_id, + # ct_id=entry.ct_id)) + + # Only install flows to dp_ports that are mapped to the vm + entries.extend(self.isltable.get_entries(vm_id=entry.vm_id, + dp_id=entry.dp_id, ct_id=entry.ct_id)) # Replace the VM port with the datapath port @@ -801,9 +810,12 @@ def register_route_mod(self, rm): elif rm.get_mod() in (RMT_ADD, RMT_DELETE): rms.extend(translator.handle_route_mod(entry, rm)) - remote_dps = self.isltable.get_entries(rem_ct=entry.ct_id, + #TODO: Restore this. This change used for testing only + #remote_dps = self.isltable.get_entries(rem_ct=entry.ct_id, + # rem_id=entry.dp_id) + remote_dps = self.isltable.get_entries(vm_id=entry.vm_id, + rem_ct=entry.ct_id, rem_id=entry.dp_id) - for r in remote_dps: if r.get_status() == RFISL_ACTIVE: local_rm = copy.deepcopy(rm) @@ -848,6 +860,10 @@ def register_dp_port(self, ct_id, dp_id, dp_port): # If there's an idle VM entry matching configuration, associate elif entry.get_status() == RFENTRY_IDLE_VM_PORT: action = REGISTER_ASSOCIATED + # This action used to update proxy's table when it comes back from + # a reset + elif entry.get_status() == RFENTRY_ACTIVE: + action = REGISTER_ACTIVE # Apply action if action == REGISTER_IDLE: @@ -855,12 +871,14 @@ def register_dp_port(self, ct_id, dp_id, dp_port): dp_port=dp_port)) self.log.info("Registering datapath port as idle (dp_id=%s, " "dp_port=%i)" % (format_id(dp_id), dp_port)) - elif action == REGISTER_ASSOCIATED: + elif action == REGISTER_ASSOCIATED: entry.associate(dp_id, dp_port, ct_id) # We can ACTIVATE a mapping with info from VM ports table # without the need of PortMap msg - vm_port_info = self.vmporttable.get_vm_port_info(vm_id=entry.vm_id, vm_port=entry.vm_port) + vm_port_info = \ + self.vmporttable.get_vm_port_info(vm_id=entry.vm_id, + vm_port=entry.vm_port) if vm_port_info is not None and vm_port_info.vs_id is not None: entry.activate(vs_id=vm_port_info.vs_id, vs_port=vm_port_info.vs_port) @@ -869,6 +887,9 @@ def register_dp_port(self, ct_id, dp_id, dp_port): dp_id=entry.dp_id, dp_port=entry.dp_port, vs_id=entry.vs_id, vs_port=entry.vs_port, operation_id=DCT_MAP_ADD) self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + + # The PortConfig used for getting config messages for the + # switch msg = PortConfig(vm_id=entry.vm_id, vm_port=entry.vm_port, operation_id=PCT_MAP_SUCCESS) self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) @@ -881,7 +902,21 @@ def register_dp_port(self, ct_id, dp_id, dp_port): entry.vm_port)) elif action == REGISTER_ISL: self._register_islconf(islconfs, ct_id, dp_id, dp_port) - + + # This is to fix the problem when Proxy comes back after it's been reset + # proxy lost its table, but rfserver is not gonna send DataPlanMap msg + elif action == REGISTER_ACTIVE: + msg = DataPlaneMap(ct_id=entry.ct_id, + dp_id=entry.dp_id, dp_port=entry.dp_port, + vs_id=entry.vs_id, vs_port=entry.vs_port, + operation_id=DCT_MAP_ADD) + self.ipc.send(RFSERVER_RFPROXY_CHANNEL, str(entry.ct_id), msg) + + # The PortConfig used for getting config messages for the + # switch + msg = PortConfig(vm_id=entry.vm_id, vm_port=entry.vm_port, + operation_id=PCT_MAP_SUCCESS) + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) def _register_islconf(self, c_entries, ct_id, dp_id, dp_port): for conf in c_entries: @@ -1000,10 +1035,11 @@ def reset_vm_port(self, vm_id, vm_port): # PortMap methods def map_port(self, vm_id, vm_port, vs_id, vs_port): - # Check if the port is already in VM Ports Table, Update the port entry if found + # Check if the port is already in VM Ports Table, + # Update the port entry if found entry = self.vmporttable.get_vm_port_info(vm_id = vm_id, vm_port = vm_port) if entry is not None: - entry.update_vs(vs_id = vs_id, vs_port = vs_port) + entry.update_vs(vs_id=vs_id, vs_port=vs_port) self.vmporttable.set_entry(entry) #TODO: Because we keep VS info, so no need RFClient to send this msg anymore # When register dp port, we can get vm and vs info to update rftable diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py index c6d31300..7b25a69b 100755 --- a/rfserver/rfservercli.py +++ b/rfserver/rfservercli.py @@ -207,10 +207,10 @@ def view_config(self, **kwargs): def view_vmports(self, **kwargs): rfserver = self.app.rfserver entries = json.loads(rfserver.get_rfvmports()) - self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n\n".\ + self.app.stdout.write("{:<18} {:<8} {:<22} {:<8} {:<18}\n\n".\ format('vm_id', 'vm_port', 'vs_id', 'vs_port','eth_addr')) for entry in entries: - self.app.stdout.write("{:<18} {:<8} {:<8} {:<8} {:<8}\n".\ + self.app.stdout.write("{:<18} {:<8} {:<22} {:<8} {:<18}\n".\ format(format_id(entry['vm_id']), entry['vm_port'], entry['vs_id'], entry['vs_port'], entry['eth_addr'])) diff --git a/rfserver/rfserverrpc.py b/rfserver/rfserverrpc.py index 525cf7c1..66ac2075 100755 --- a/rfserver/rfserverrpc.py +++ b/rfserver/rfserverrpc.py @@ -32,6 +32,22 @@ def get_rfconfig(self): return json.dumps(results) + def get_rfislconfig(self): + entries = self.rfserver.islconf.get_entries() + results = [] + for entry in entries: + results.append(entry.to_dict()) + + return json.dumps(results) + + def get_rfisltable(self): + entries = self.rfserver.isltable.get_entries() + results = [] + for entry in entries: + results.append(entry.to_dict()) + + return json.dumps(results) + def get_rfvmports(self): entries = self.rfserver.vmporttable.get_entries() results = [] From 79c9af69fddaf0d11c34e6c7cfadf170c23f53c1 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Fri, 9 Jan 2015 14:52:25 +1300 Subject: [PATCH 21/52] Basically working. Included with two test scripts. The scripts require some preprations for LXC container beforehand. Signed-off-by: Trung Truong --- dynamic-test2.sh | 272 ++++++++++++++++++++++++++++++++++++++++ mytest/dynamic-test2.py | 147 ++++++++++++++++++++++ rfserver/rfserver.py | 31 ++++- rfserver/rfservercli.py | 36 ++++++ rfserver/rfserverrpc.py | 2 +- 5 files changed, 485 insertions(+), 3 deletions(-) create mode 100755 dynamic-test2.sh create mode 100755 mytest/dynamic-test2.py diff --git a/dynamic-test2.sh b/dynamic-test2.sh new file mode 100755 index 00000000..0f590b19 --- /dev/null +++ b/dynamic-test2.sh @@ -0,0 +1,272 @@ +#!/bin/bash + +# Set to 0 to use external switch(es) +STARTBVMS=0 + +MULTITABLEDPS="''" +SATELLITEDPS="''" + +HOME=/home/ubuntu +RF_HOME=$HOME/RouteFlow +RFSERVERCONFIG=/tmp/rfserverconfig.csv +RFSERVERINTERNAL=/tmp/rfserverinternal.csv +HOME_RFSERVERCONFIG="$HOME/"`basename $RFSERVERCONFIG` +HOME_RFSERVERINTERNAL="$HOME/"`basename $RFSERVERINTERNAL` +CONTROLLER_PORT=6633 +LXCDIR=/var/lib/lxc +RFBR=br0 +RFDP=dp0 +RFDPID=7266767372667673 +OFP=OpenFlow13 +HOSTVMIP=192.168.10.1 +VSCTL="ovs-vsctl" +OFCTL="ovs-ofctl -O$OFP" +export PATH=$PATH:/usr/local/bin:/usr/local/sbin +export PYTHONPATH=$PYTHONPATH:$RF_HOME + +#modprobe 8021q +ulimit -c 1000000000 + +if [ "$EUID" != "0" ]; then + echo "You must be root to run this script." + exit 1 +fi + +ACTION="" +case "$1" in +--ryu) + ACTION="RYU" + ;; +--reset) + ACTION="RESET" + ;; +*) + echo "Invalid argument: $1" + echo "Options: " + echo " --ryu: run using RYU" + echo " --reset: stop running and clear data from previous executions" + exit + ;; +esac + +cd $RF_HOME + +wait_port_listen() { + port=$1 + while ! `nc -z localhost $port` ; do + echo -n . + sleep 1 + done +} + +echo_bold() { + echo -e "\033[1m${1}\033[0m" +} + +kill_process_tree() { + top=$1 + pid=$2 + + children=`ps -o pid --no-headers --ppid ${pid}` + + for child in $children + do + kill_process_tree 0 $child + done + + if [ $top -eq 0 ]; then + kill -9 $pid &> /dev/null + fi +} + +add_local_br() { + br=$1 + dpid=$2 + $VSCTL add-br $br + $VSCTL set bridge $br protocols=$OFP + if [ "$dpid" != "" ] ; then + $VSCTL set bridge $br other-config:datapath-id=$dpid + fi + ifconfig $br up + check_local_br_up $br +} + +check_local_br_up() { + br=$1 + echo waiting for OVS sw/controller $br to come up + while ! $OFCTL ping $br 64|grep -q "64 bytes from" ; do + echo -n "." + sleep 1 + done +} + +start_ovs() { + if [ ! -f /usr/local/etc/openvswitch/conf.db ] ; then + ovsdb-tool create /usr/local/etc/openvswitch/conf.db /usr/local/share/openvswitch/vswitch.ovsschema + fi + ovsdb-server --pidfile --detach --remote=punix:$OVSSOCK + ovs-vswitchd --pidfile --detach unix:$OVSSOCK +} + +start_rfvms() { + i=0 + for vm in rfvmA rfvmB rfvmC rfvmD; do + i=$(($i+1)) + ROOTFS=$LXCDIR/$vm/rootfs + # Prepare configs for LXCs + cat > $ROOTFS/etc/network/interfaces <> $ROOTFS/etc/network/interfaces< $ROOTFS/etc/rc.local < $ROOTFS/root/run_rfclient.sh < /var/log/rfclient.log 2> /var/log/rfclient.log.err +EOF + chmod +x $ROOTFS/root/run_rfclient.sh + + cp /dev/null $ROOTFS/var/log/syslog + VMLOG=/tmp/$vm.log + rm -f $VMLOG + lxc-start -n $vm -l DEBUG -o $VMLOG -d + done +} + +stop_rfvms() { + echo_bold "-> Stopping the virtual machines..." + for vm in rfvmA rfvmB rfvmC rfvmD; do + lxc-stop -n $vm &> /dev/null; + ROOTFS=$LXCDIR/$vm/rootfs + rm -rf $ROOTFS/var/run/network/ifstate; + done +} + +reset() { + echo_bold "-> Stopping and resetting LXC VMs..."; + stop_rfvms + + init=$1; + if [ $init -eq 1 ]; then + echo_bold "-> Starting OVS daemons..."; + #start_ovs + else + echo_bold "-> Stopping child processes..."; + kill_process_tree 1 $$ + fi + + sudo $VSCTL del-br $RFBR &> /dev/null; + sudo $VSCTL del-br $RFDP &> /dev/null; + sudo $VSCTL emer-reset &> /dev/null; +} +reset 1 +trap "reset 0; exit 0" INT + +if [ "$ACTION" != "RESET" ]; then + if [ -f "$HOME_RFSERVERCONFIG" ] && [ -f "$HOME_RFSERVERINTERNAL" ] ; then + echo_bold "-> Using existing external config..." + cp $HOME_RFSERVERCONFIG $RFSERVERCONFIG + cp $HOME_RFSERVERINTERNAL $RFSERVERINTERNAL + else + echo_bold "-> Run with default config..." + cp /dev/null > $RFSERVERCONFIG + echo "vm_id,vm_port,ct_id,dp_id,dp_port" > $RFSERVERCONFIG + + # Initially, rfvmA is mapped the port1 to dp1, port1 + # & port2 to dp2, port1 + #echo 0x2a0a0a0a0a0,1,0,0x01,1 >> $RFSERVERCONFIG + #echo 0x2a0a0a0a0a0,2,0,0x02,1 >> $RFSERVERCONFIG + + cp /dev/null $RFSERVERINTERNAL + echo "vm_id,ct_id,dp_id,dp_port,eth_addr,rem_ct,rem_id,rem_port,rem_eth_addr" > $RFSERVERINTERNAL + echo 0x2a0a0a0a0a0,0,0x01,2,02:a1:a1:a1:a1:a1,0,0x02,2,02:a2:a2:a2:a2:a2 >> $RFSERVERINTERNAL + + echo 0x2b0b0b0b0b0,0,0x01,3,02:b1:b1:b1:b1:b1,0,0x02,3,02:b2:b2:b2:b2:b2 >> $RFSERVERINTERNAL + + echo 0x2c0c0c0c0c0,0,0x01,4,02:c1:c1:c1:c1:c1,0,0x02,4,02:c2:c2:c2:c2:c2 >> $RFSERVERINTERNAL + + echo 0x2d0d0d0d0d0,0,0x01,5,02:d1:d1:d1:d1:d1,0,0x03,1,02:d2:d2:d2:d2:d2 >> $RFSERVERINTERNAL + echo 0x2d0d0d0d0d0,0,0x03,2,02:d1:d1:d1:d1:d1,0,0x02,5,02:d2:d2:d2:d2:d2 >> $RFSERVERINTERNAL + fi + + echo_bold "-> Starting the management network ($RFBR)..." + add_local_br $RFBR + ifconfig $RFBR $HOSTVMIP + + echo_bold "-> Starting RFServer..." + #winpdb ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + + echo_bold "-> Starting the controller ($ACTION) and RFPRoxy..." + case "$ACTION" in + RYU) + cd .. + cd ryu-rfproxy + ryu-manager --use-stderr --ofp-tcp-listen-port=$CONTROLLER_PORT ryu-rfproxy/rfproxy.py & + ;; + esac + cd - &> /dev/null + wait_port_listen $CONTROLLER_PORT + check_local_br_up tcp:127.0.0.1:$CONTROLLER_PORT + + echo_bold "-> Starting the control plane network ($RFDP VS)..." + $VSCTL add-br $RFDP + $VSCTL set bridge $RFDP other-config:datapath-id=$RFDPID + $VSCTL set bridge $RFDP protocols=$OFP + $VSCTL set-controller $RFDP tcp:127.0.0.1:$CONTROLLER_PORT + $OFCTL add-flow $RFDP actions=CONTROLLER:65509 + ifconfig $RFDP up + check_local_br_up $RFDP + + echo_bold "-> Waiting for $RFDP to connect to controller..." + while ! $VSCTL find Controller target=\"tcp:127.0.0.1:$CONTROLLER_PORT\" is_connected=true | grep -q connected ; do + echo -n . + sleep 1 + done + + echo_bold "-> Starting virtual machines..." + start_rfvms + while ! ifconfig -s rfvmA.0 ; do + echo -n . + sleep 1 + done + + # Add VM eth0 port to management bridge + for vm in rfvmA rfvmB rfvmC rfvmD; do + $VSCTL add-port $RFBR $vm.0 + done + + # Add VM interfaces to dataplane bridge + for i in `netstat -i|grep rfvm|cut -f 1 -d " "` ; do + if [ "$i" != "`echo $i | grep .0`" ] ; then + $VSCTL add-port $RFDP $i + fi + done + + echo_bold "-> Waiting for VMs to come up..." + while ! ping -W 1 -c 1 192.168.10.101 ; do + echo -n . + sleep 1 + done + + echo_bold "You can stop this test by pressing Ctrl+C." + wait +fi +exit 0 diff --git a/mytest/dynamic-test2.py b/mytest/dynamic-test2.py new file mode 100755 index 00000000..c54677ca --- /dev/null +++ b/mytest/dynamic-test2.py @@ -0,0 +1,147 @@ +#!/usr/bin/python +import sys +from mininet.net import Mininet +from mininet.node import Controller, RemoteController +from mininet.node import CPULimitedHost, Host, Node +from mininet.node import OVSSwitch +from mininet.cli import CLI +from mininet.log import setLogLevel, info +from mininet.link import TCLink +from mininet.util import run + +import time +import xmlrpclib +import json + +# +# This will create two switches with one inter-link +# It takes input argument as the number of host ports for each sw +# the topology will look like +# +# ____ ____ +# | |(2)-----1Mb,10ms------(2)| | +# h1----(1)| s1 |(3)----10Mb,10ms------(3)| s2 |(1)-----h2 +# |____|(4)----20Mb,10ms------(4)|____| +# (5) (5) +# | ____ | +# | | | | +# \--------(1)| s3 |(2)--------/ +# |____| +# +def myNetwork(): + + rfserver_ip = '192.168.56.101' + # Initialize RPC connection to RFServer. + # + info("Connect to RFServer...\n") + rfserver = xmlrpclib.ServerProxy('http://' + '192.168.56.101' +':8008', + allow_none=True) + + info("Prepare network configuration for the tests...\n") + net = Mininet(topo=None, build=False, link=TCLink, switch=OVSSwitch) + info('---------- Add controller ----------\n') + c0 = net.addController(name='c0', controller=RemoteController, + ip=rfserver_ip, port=6633) + + info('---------- Add switches\n ----------') + s1 = net.addSwitch(name='s1', protocols='OpenFlow13') + s2 = net.addSwitch(name='s2', protocols='OpenFlow13') + s3 = net.addSwitch(name='s3', protocols='OpenFlow13') + + info('*** Add hosts\n') + h1 = net.addHost(name="h1", cls=Host, ip='172.16.1.2/24', defaultRoute='via 172.16.1.1') + h2 = net.addHost(name="h2", cls=Host, ip='172.16.2.2/24', defaultRoute='via 172.16.2.1') + + info("---------- Add links ----------\n") + net.addLink(h1, s1) + net.addLink(h2, s2) + + net.addLink(s1, s2, bw=1, delay='10ms') + net.addLink(s1, s2, bw=10, delay='10ms') + net.addLink(s1, s2, bw=20, delay='10ms') + net.addLink(s1, s3) + net.addLink(s2, s3) + + info('---------- Start network ----------\n') + net.build() + net.start() + run("ovs-vsctl set bridge s1 protocols=OpenFlow13") + run("ovs-vsctl set bridge s2 protocols=OpenFlow13") + run("ovs-vsctl set bridge s3 protocols=OpenFlow13") + + info('\n\n') + info('Start the first test: Two ISL linked witches are mapped to rfvmA\n') + first_test(rfserver, net) + info('\n\n') + info('Start the first test: Two ISL linked witches are mapped to rfvmB\n') + second_test(rfserver, net) + info('\n\n') + info('Start the first test: Two ISL linked witches are mapped to rfvmC\n') + third_test(rfserver, net) + try: + while True: + pass + except KeyboardInterrupt: + pass + #CLI(net) + net.stop() + +def first_test(rfserver, net): + info('Test in progress....\n') + info('Two switches with an ISL link are mapped to rfvmA\n') + config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 1, 'dp_id': '1', 'dp_port': 1}, + {'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 1}] + + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + + net.get('h1').cmd('ping -c 1 172.16.1.1') + net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') + time.sleep(10) + info(net.iperf()) + info('\n') + rfserver.delete_map_configs(config[0]['vm_id']) + info('Test completed') + +def second_test(rfserver, net): + info('Test in progress....\n') + config = [{'vm_id': '2b0b0b0b0b0', 'vm_port': 1, 'dp_id': '1', 'dp_port': 1}, + {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 1}] + + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + + net.get('h1').cmd('ping -c 1 172.16.1.1') + net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') + time.sleep(10) + info(net.iperf()) + info('\n') + rfserver.delete_map_configs(config[0]['vm_id']) + info('Test completed\n') + +def third_test(rfserver, net): + info('Test in progress....\n') + info('Two switches with an ISL link are mapped to rfvmC\n') + config = [{'vm_id': '2c0c0c0c0c0', 'vm_port': 1, 'dp_id': '1', 'dp_port': 1}, + {'vm_id': '2c0c0c0c0c0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 1}] + + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + + net.get('h1').cmd('ping -c 1 172.16.1.1') + net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') + time.sleep(10) + info(net.iperf()) + info('\n') + rfserver.delete_map_configs(config[0]['vm_id']) + info('Test completed\n') + +if __name__ == '__main__': + setLogLevel('info') + myNetwork() \ No newline at end of file diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index e8777f01..c967cfea 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -156,6 +156,12 @@ def delete_flows_by_port(self, entry): rm.set_outport(entry.dp_port) rm.add_option(self.DEFAULT_PRIORITY) rms.append(rm) + + rm = RouteMod(RMT_DELETE, self.dp_id) + rm.add_match(Match.ETHERNET(entry.eth_addr)) + rm.add_option(self.DEFAULT_PRIORITY) + rms.append(rm) + return rms class SatelliteRouteModTranslator(DefaultRouteModTranslator): @@ -579,6 +585,14 @@ def delete_map_configs(self, **kwargs): (format_id(rf_entry.dp_id), rf_entry.dp_port, format_id(rf_entry.vs_id), rf_entry.vs_port)) + + self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(rf_entry.vm_id), + PortConfig(vm_id=rf_entry.vm_id, + vm_port=rf_entry.vm_port, + operation_id=PCT_RESET)) + self.log.info("Resetting client port (vm_id=%s, vm_port=%i)" % + (format_id(rf_entry.vm_id), rf_entry.vm_port)) + # Now update switch flow table translator = self.route_mod_translator[rf_entry.dp_id] rms = translator.delete_flows_by_port(rf_entry) @@ -876,6 +890,10 @@ def register_dp_port(self, ct_id, dp_id, dp_port): # We can ACTIVATE a mapping with info from VM ports table # without the need of PortMap msg + # Don't need this temporarily, because I'm not gonna disable PortMap + # now. As I keep all vm_port info, PortMap is not necessary. But disable + # it later + """ vm_port_info = \ self.vmporttable.get_vm_port_info(vm_id=entry.vm_id, vm_port=entry.vm_port) @@ -893,7 +911,7 @@ def register_dp_port(self, ct_id, dp_id, dp_port): msg = PortConfig(vm_id=entry.vm_id, vm_port=entry.vm_port, operation_id=PCT_MAP_SUCCESS) self.ipc.send(RFCLIENT_RFSERVER_CHANNEL, str(entry.vm_id), msg) - + """ self.rftable.set_entry(entry) self.log.info("Registering datapath port and associating to " "client port (dp_id=%s, dp_port=%i, vm_id=%s, " @@ -979,6 +997,11 @@ def config_dp(self, ct_id, dp_id): if is_rfvs(dp_id): return True else: + #TODO: Trung: Not quite sure when we need to check this + # with the first dp port register msg comes in, this "if" statement + # will false, the subsequences are OK, then we create RM translator + # I'm going to change this, so it can work when run RF with empty + # config??? if (self.rftable.is_dp_registered(ct_id, dp_id) or self.isltable.is_dp_registered(ct_id, dp_id)): if dp_id not in self.route_mod_translator: @@ -1037,13 +1060,17 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): # Check if the port is already in VM Ports Table, # Update the port entry if found + # Disable this code portion for now, because it causes an issue with dp + # comming back from a reset (when restart Mininet, no flow entries are + # installed + """ entry = self.vmporttable.get_vm_port_info(vm_id = vm_id, vm_port = vm_port) if entry is not None: entry.update_vs(vs_id=vs_id, vs_port=vs_port) self.vmporttable.set_entry(entry) #TODO: Because we keep VS info, so no need RFClient to send this msg anymore # When register dp port, we can get vm and vs info to update rftable - + """ entry = None entry = self.rftable.get_entry_by_vm_port(vm_id, vm_port) if entry is not None and entry.get_status() == RFENTRY_ASSOCIATED: diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py index 7b25a69b..9d9c41e1 100755 --- a/rfserver/rfservercli.py +++ b/rfserver/rfservercli.py @@ -228,11 +228,47 @@ def view_dpports(self, **kwargs): entry['dp_id'], entry['dp_port'])) self.app.stdout.write("\n") + +class SaveCommand(Command): + log = logging.getLogger(__name__) + + def get_parser(self, prog_name): + parser = super(SaveCommand, self).get_parser(prog_name) + parser.add_argument('-config', '--c', type=str, + required=None, default=None) + + parser.add_argument('-isl', '--i', type=str, + required=None, default=None) + + return parser + + def take_action(self, parsed_args): + config_path = None if parsed_args.config is None \ + else str(parsed_args.config) + + isl_path = None if parsed_args.isl is None \ + else str(parsed_args.isl) + + if self.app.rfconfig_path is None or self.app.islconf_path is None: + self.app.log.inf("Please provide filename once") + return + if config_path is not None: + self.app.rfconfig_path = config_path + if isl_path is not None: + self.app.islconf_path = isl_path + + rfserver = self.app.rfserver + entries = json.loads(rfserver.get_rfconfig()) + entries = json.loads(rfserver.get_rfislconfig()) + + class RFServerCLI(App): log = logging.getLogger(__name__) log.setLevel(logging.INFO) def __init__(self): + self.rfconfig_path = None + self.islconf_path = None self.rfserver = xmlrpclib.ServerProxy('http://localhost:8008', allow_none=True) command = CommandManager('RFServer') super(RFServerCLI, self).__init__( diff --git a/rfserver/rfserverrpc.py b/rfserver/rfserverrpc.py index 66ac2075..43602981 100755 --- a/rfserver/rfserverrpc.py +++ b/rfserver/rfserverrpc.py @@ -6,7 +6,7 @@ class RFServerRPC(): def __init__(self, rfserver): self.rfserver = rfserver - self.rpcserver = SimpleXMLRPCServer(("localhost", 8008), allow_none=True) + self.rpcserver = SimpleXMLRPCServer(("", 8008), allow_none=True) self.rpcserver.register_instance(RPC_processor(self.rfserver)) print "RPC server started" From bb4de043e0943009eba4191c76a016f696340a8f Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Fri, 9 Jan 2015 15:09:08 +1300 Subject: [PATCH 22/52] New developements for dynamic mapping (RFClient part) will be based on Kris libnl work) Add build.sh (downloaded from Kris libnl) Signed-off-by: Trung Truong --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index c8079143..2673ce8c 100755 --- a/build.sh +++ b/build.sh @@ -20,8 +20,8 @@ PIP_OPTS="" ROUTEFLOW_GIT="https://github.com/routeflow/RouteFlow.git" DEPENDENCIES="build-essential git-core libboost-dev libboost-dev \ libboost-program-options-dev libboost-thread-dev \ - libboost-filesystem-dev libboost-system-dev iproute-dev python-dev \ - python-pip python-bson" + libboost-filesystem-dev libboost-system-dev libnl-3-dev libnl-route-3-dev \ + python-dev python-pip python-bson" usage() { echo "usage:$0 [-hcqvdsgiu] [-m MONGO_VERSION] [-o OVS_VERSION]" \ From 11dd62aa2ade8a33f597694c299437f7c4794d40 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Fri, 9 Jan 2015 23:34:19 +1300 Subject: [PATCH 23/52] Change test shell script to automatically copy rfclient to LXC Signed-off-by: Trung Truong --- dynamic-test2.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dynamic-test2.sh b/dynamic-test2.sh index 0f590b19..5dd55a88 100755 --- a/dynamic-test2.sh +++ b/dynamic-test2.sh @@ -8,6 +8,7 @@ SATELLITEDPS="''" HOME=/home/ubuntu RF_HOME=$HOME/RouteFlow +RYU_HOME=$HOME/ryu-rfproxy RFSERVERCONFIG=/tmp/rfserverconfig.csv RFSERVERINTERNAL=/tmp/rfserverinternal.csv HOME_RFSERVERCONFIG="$HOME/"`basename $RFSERVERCONFIG` @@ -141,6 +142,13 @@ EOF #!/bin/sh /opt/rfclient/rfclient > /var/log/rfclient.log 2> /var/log/rfclient.log.err EOF + # Create the rfclient dir + RFCLIENTDIR=$ROOTFS/opt/rfclient + mkdir -p $RFCLIENTDIR + # Copy the rfclient executable + cp rfclient/rfclient $RFCLIENTDIR/rfclient + #cp -p -P /usr/local/lib/libzmq* $ROOTFS/usr/local/lib + chroot $ROOTFS ldconfig chmod +x $ROOTFS/root/run_rfclient.sh cp /dev/null $ROOTFS/var/log/syslog @@ -151,6 +159,7 @@ EOF } stop_rfvms() { + cd $RF_HOME echo_bold "-> Stopping the virtual machines..." for vm in rfvmA rfvmB rfvmC rfvmD; do lxc-stop -n $vm &> /dev/null; @@ -217,8 +226,7 @@ if [ "$ACTION" != "RESET" ]; then echo_bold "-> Starting the controller ($ACTION) and RFPRoxy..." case "$ACTION" in RYU) - cd .. - cd ryu-rfproxy + cd $RYU_HOME ryu-manager --use-stderr --ofp-tcp-listen-port=$CONTROLLER_PORT ryu-rfproxy/rfproxy.py & ;; esac From 466233ba4529f9b55283ff00805fb5aa903fe6e4 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Fri, 9 Jan 2015 23:47:13 +1300 Subject: [PATCH 24/52] Some editing on projectw.sh Signed-off-by: Trung Truong --- projectw.sh | 101 +++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/projectw.sh b/projectw.sh index 476f4b14..7870676d 100755 --- a/projectw.sh +++ b/projectw.sh @@ -170,68 +170,66 @@ EOF # Configure the VM cat > $RFVM1/config <> $RFVM1/config< $ROOTFS/etc/network/interfaces <> $RFVM1/rootfs/etc/network/interfaces<> $RFVM1/rootfs/etc/network/interfaces< $ROOTFS/etc/rc.local < $ROOTFS/root/run_rfclient.sh < /var/log/rfclient.log 2> /var/log/rfclient.log.err +#!/bin/sh +/opt/rfclient/rfclient > /var/log/rfclient.log 2> /var/log/rfclient.log.err EOF chmod +x $RFVM1/rootfs/root/run_rfclient.sh @@ -258,7 +256,6 @@ EOF # Create the rfclient dir RFCLIENTDIR=$ROOTFS/opt/rfclient mkdir -p $RFCLIENTDIR - # Copy the rfclient executable cd $RF_HOME cp rfclient/rfclient $RFCLIENTDIR/rfclient From 1e78ad482e6df70476c57b362bc580ee33068e4c Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 00:07:54 +1300 Subject: [PATCH 25/52] change LXC config file, in order to fix LXC hang up issue Signed-off-by: Trung Truong --- projectw.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/projectw.sh b/projectw.sh index 7870676d..ca3cf482 100755 --- a/projectw.sh +++ b/projectw.sh @@ -170,9 +170,13 @@ EOF # Configure the VM cat > $RFVM1/config < Date: Sat, 10 Jan 2015 00:31:44 +1300 Subject: [PATCH 26/52] Fix minor error in test script Signed-off-by: Trung Truong --- dynamic-test2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamic-test2.sh b/dynamic-test2.sh index 5dd55a88..5b3cdd3f 100755 --- a/dynamic-test2.sh +++ b/dynamic-test2.sh @@ -195,7 +195,7 @@ if [ "$ACTION" != "RESET" ]; then cp $HOME_RFSERVERINTERNAL $RFSERVERINTERNAL else echo_bold "-> Run with default config..." - cp /dev/null > $RFSERVERCONFIG + cp /dev/null $RFSERVERCONFIG echo "vm_id,vm_port,ct_id,dp_id,dp_port" > $RFSERVERCONFIG # Initially, rfvmA is mapped the port1 to dp1, port1 From c2af1c509015dcd36704cd394d61d040b7e76834 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 19:04:13 +1300 Subject: [PATCH 27/52] Copy Chris FlowTable.cc code because somehow merge branch not work as expected Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 96 +++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 62 deletions(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 8c7a381c..7bf67f4e 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -91,38 +91,16 @@ FlowTable::FlowTable(const FlowTable& other) { } void FlowTable::operator()() { - rtnl_open(&rthNeigh, RTMGRP_NEIGH); - int rs = setsockopt(rthNeigh.fd, SOL_SOCKET, SO_RCVBUFFORCE, &nl_buffersize, sizeof(nl_buffersize)); - if (rs != 0) { - syslog(LOG_CRIT, "cannot set socket size for neighbors: %d", errno); - exit(rs); - } - HTPolling = boost::thread(&rtnl_listen, &rthNeigh, &HTPollingCb, this); - - switch (this->source) { - case RS_NETLINK: { - syslog(LOG_NOTICE, "Netlink interface enabled"); - rtnl_open(&rth, RTMGRP_IPV4_ROUTE | RTMGRP_IPV6_ROUTE | - RTMGRP_IPV4_MROUTE | RTMGRP_IPV6_MROUTE); - int rs = setsockopt(rth.fd, SOL_SOCKET, SO_RCVBUFFORCE, &nl_buffersize, sizeof(nl_buffersize)); - if (rs != 0) { - syslog(LOG_CRIT, "cannot set socket size for routes: %d", errno); - exit(rs); - } - RTPolling = boost::thread(&rtnl_listen, &rth, &RTPollingCb, this); - break; - } - case RS_FPM: { - FPMServer *fpm = new FPMServer(this); - RTPolling = boost::thread(*fpm); - syslog(LOG_NOTICE, "FPM interface enabled"); - break; - } - default: { - syslog(LOG_CRIT, "Invalid route source specified. Disabling route updates."); - break; - } - } + switch (this->source) { + case RS_NETLINK: { + initNLListener(); + break; + } + default: { + syslog(LOG_CRIT, "Invalid route source specified. Disabling route updates."); + break; + } + } GWResolver = boost::thread(&FlowTable::GWResolverCb, this); GWResolver.join(); @@ -275,37 +253,31 @@ void FlowTable::GWResolverCb(FlowTable *ft) { break; } } - if (ft->unresolvedRoutes.size() > 0) { - set resolvedRoutes; - set::iterator it; - uint64_t gatewaysResolved = 0; - for (it = ft->unresolvedRoutes.begin(); it != ft->unresolvedRoutes.end(); ++it) { - const RouteEntry& re = ft->routeTable[*it]; - const string addr_str = re.address.toString(); - const string mask_str = re.netmask.toString(); - const string gw_str = re.gateway.toString(); - if (ft->findHost(re.gateway) == MAC_ADDR_NONE) { - syslog(LOG_DEBUG, - "Still cannot resolve gateway %s, will retry route %s/%s", - gw_str.c_str(), addr_str.c_str(), mask_str.c_str()); - ft->resolveGateway(re.gateway, re.interface); - } else { - syslog(LOG_DEBUG, - "Adding previously unresolved route %s/%s via %s", - addr_str.c_str(), mask_str.c_str(), gw_str.c_str()); - ft->sendToHw(RMT_ADD, re); - resolvedRoutes.insert(*it); - ++gatewaysResolved; - } - } - if (gatewaysResolved) { - for (it = resolvedRoutes.begin(); it != resolvedRoutes.end(); ++it) { - ft->unresolvedRoutes.erase(*it); - } + + set resolvedRoutes; + set::iterator it; + for (it = ft->unresolvedRoutes.begin(); it != ft->unresolvedRoutes.end(); ++it) { + const RouteEntry& re = ft->routeTable[*it]; + const string addr_str = re.address.toString(); + const string mask_str = re.netmask.toString(); + const string gw_str = re.gateway.toString(); + if (ft->findHost(re.gateway) == MAC_ADDR_NONE) { + syslog(LOG_DEBUG, + "Still cannot resolve gateway %s, will retry route %s/%s", + gw_str.c_str(), addr_str.c_str(), mask_str.c_str()); + ft->resolveGateway(re.gateway, re.interface); + } else { + syslog(LOG_DEBUG, + "Adding previously unresolved route %s/%s via %s", + addr_str.c_str(), mask_str.c_str(), gw_str.c_str()); + ft->sendToHw(RMT_ADD, re); + resolvedRoutes.insert(*it); } - } else { - usleep(1000); - } + } + for (it = resolvedRoutes.begin(); it != resolvedRoutes.end(); ++it) { + ft->unresolvedRoutes.erase(*it); + } + usleep(100); } } From 34872447fdce0996ce213296de60aaec22324efb Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 19:43:24 +1300 Subject: [PATCH 28/52] Correct eth_add to eth_addr Signed-off-by: Trung Truong --- rfserver/rftable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfserver/rftable.py b/rfserver/rftable.py index 08f654a4..69ef1066 100644 --- a/rfserver/rftable.py +++ b/rfserver/rftable.py @@ -524,7 +524,7 @@ def update_vs(self, vs_id, vs_port): self.vs_port = vs_port def update_eth_addr(self, eth_addr): - self.eth_addr = eth_add + self.eth_addr = eth_addr def to_dict(self): data = {} From 3f01f7c0ecc5c026f3a8637fd42b31ec3e81034c Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 21:41:58 +1300 Subject: [PATCH 29/52] Added dynamic-test3 two VMs, each has two ports, are mapped to two switches. two sw have 3 inter-links, & each has a port connected to a host there are 3 tests: each uses one inter-link Signed-off-by: Trung Truong --- dynamic-test3.sh | 283 ++++++++++++++++++++++++++++++++++++++++ mytest/dynamic-test3.py | 156 ++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100755 dynamic-test3.sh create mode 100755 mytest/dynamic-test3.py diff --git a/dynamic-test3.sh b/dynamic-test3.sh new file mode 100755 index 00000000..12b40f84 --- /dev/null +++ b/dynamic-test3.sh @@ -0,0 +1,283 @@ +#!/bin/bash + +# Set to 0 to use external switch(es) +STARTBVMS=0 + +MULTITABLEDPS="''" +SATELLITEDPS="''" + +HOME=/home/ubuntu +RF_HOME=$HOME/RouteFlow +RYU_HOME=$HOME/ryu-rfproxy +RFSERVERCONFIG=/tmp/rfserverconfig.csv +RFSERVERINTERNAL=/tmp/rfserverinternal.csv +HOME_RFSERVERCONFIG="$HOME/"`basename $RFSERVERCONFIG` +HOME_RFSERVERINTERNAL="$HOME/"`basename $RFSERVERINTERNAL` +CONTROLLER_PORT=6633 +LXCDIR=/var/lib/lxc +RFBR=br0 +RFDP=dp0 +RFDPID=7266767372667673 +OFP=OpenFlow13 +HOSTVMIP=192.168.10.1 +VSCTL="ovs-vsctl" +OFCTL="ovs-ofctl -O$OFP" +export PATH=$PATH:/usr/local/bin:/usr/local/sbin +export PYTHONPATH=$PYTHONPATH:$RF_HOME + +#modprobe 8021q +ulimit -c 1000000000 + +if [ "$EUID" != "0" ]; then + echo "You must be root to run this script." + exit 1 +fi + +ACTION="" +case "$1" in +--ryu) + ACTION="RYU" + ;; +--reset) + ACTION="RESET" + ;; +*) + echo "Invalid argument: $1" + echo "Options: " + echo " --ryu: run using RYU" + echo " --reset: stop running and clear data from previous executions" + exit + ;; +esac + +cd $RF_HOME + +wait_port_listen() { + port=$1 + while ! `nc -z localhost $port` ; do + echo -n . + sleep 1 + done +} + +echo_bold() { + echo -e "\033[1m${1}\033[0m" +} + +kill_process_tree() { + top=$1 + pid=$2 + + children=`ps -o pid --no-headers --ppid ${pid}` + + for child in $children + do + kill_process_tree 0 $child + done + + if [ $top -eq 0 ]; then + kill -9 $pid &> /dev/null + fi +} + +add_local_br() { + br=$1 + dpid=$2 + $VSCTL add-br $br + $VSCTL set bridge $br protocols=$OFP + if [ "$dpid" != "" ] ; then + $VSCTL set bridge $br other-config:datapath-id=$dpid + fi + ifconfig $br up + check_local_br_up $br +} + +check_local_br_up() { + br=$1 + echo waiting for OVS sw/controller $br to come up + while ! $OFCTL ping $br 64|grep -q "64 bytes from" ; do + echo -n "." + sleep 1 + done +} + +start_ovs() { + if [ ! -f /usr/local/etc/openvswitch/conf.db ] ; then + ovsdb-tool create /usr/local/etc/openvswitch/conf.db /usr/local/share/openvswitch/vswitch.ovsschema + fi + ovsdb-server --pidfile --detach --remote=punix:$OVSSOCK + ovs-vswitchd --pidfile --detach unix:$OVSSOCK +} + +start_rfvms() { + i=1 + j=2 + for vm in rfvmA rfvmB; do + ROOTFS=$LXCDIR/$vm/rootfs + # Prepare configs for LXCs + cat > $ROOTFS/etc/network/interfaces <> $ROOTFS/etc/network/interfaces< $ROOTFS/etc/rc.local < $ROOTFS/root/run_rfclient.sh < /var/log/rfclient.log 2> /var/log/rfclient.log.err +EOF + # Create the rfclient dir + RFCLIENTDIR=$ROOTFS/opt/rfclient + mkdir -p $RFCLIENTDIR + # Copy the rfclient executable + cp rfclient/rfclient $RFCLIENTDIR/rfclient + #cp -p -P /usr/local/lib/libzmq* $ROOTFS/usr/local/lib + chroot $ROOTFS ldconfig + chmod +x $ROOTFS/root/run_rfclient.sh + + cp /dev/null $ROOTFS/var/log/syslog + VMLOG=/tmp/$vm.log + rm -f $VMLOG + lxc-start -n $vm -l DEBUG -o $VMLOG -d + done +} + +stop_rfvms() { + cd $RF_HOME + echo_bold "-> Stopping the virtual machines..." + for vm in rfvmA rfvmB; do + lxc-stop -n $vm &> /dev/null; + ROOTFS=$LXCDIR/$vm/rootfs + rm -rf $ROOTFS/var/run/network/ifstate; + done +} + +reset() { + echo_bold "-> Stopping and resetting LXC VMs..."; + stop_rfvms + + init=$1; + if [ $init -eq 1 ]; then + echo_bold "-> Starting OVS daemons..."; + #start_ovs + else + echo_bold "-> Stopping child processes..."; + kill_process_tree 1 $$ + fi + + sudo $VSCTL del-br $RFBR &> /dev/null; + sudo $VSCTL del-br $RFDP &> /dev/null; + sudo $VSCTL emer-reset &> /dev/null; +} +reset 1 +trap "reset 0; exit 0" INT + +if [ "$ACTION" != "RESET" ]; then + if [ -f "$HOME_RFSERVERCONFIG" ] && [ -f "$HOME_RFSERVERINTERNAL" ] ; then + echo_bold "-> Using existing external config..." + cp $HOME_RFSERVERCONFIG $RFSERVERCONFIG + cp $HOME_RFSERVERINTERNAL $RFSERVERINTERNAL + else + echo_bold "-> Run with default config..." + cp /dev/null $RFSERVERCONFIG + echo "vm_id,vm_port,ct_id,dp_id,dp_port" > $RFSERVERCONFIG + + # Initially, rfvmA is mapped the port1 to dp1, port1 + # & port2 to dp2, port1 + #echo 0x2a0a0a0a0a0,1,0,0x01,1 >> $RFSERVERCONFIG + #echo 0x2a0a0a0a0a0,2,0,0x02,1 >> $RFSERVERCONFIG + + cp /dev/null $RFSERVERINTERNAL + echo "vm_id,ct_id,dp_id,dp_port,eth_addr,rem_ct,rem_id,rem_port,rem_eth_addr" > $RFSERVERINTERNAL + #echo 0x2a0a0a0a0a0,0,0x01,2,02:a1:a1:a1:a1:a1,0,0x02,2,02:a2:a2:a2:a2:a2 >> $RFSERVERINTERNAL + + #echo 0x2b0b0b0b0b0,0,0x01,3,02:b1:b1:b1:b1:b1,0,0x02,3,02:b2:b2:b2:b2:b2 >> $RFSERVERINTERNAL + + #echo 0x2c0c0c0c0c0,0,0x01,4,02:c1:c1:c1:c1:c1,0,0x02,4,02:c2:c2:c2:c2:c2 >> $RFSERVERINTERNAL + + #echo 0x2d0d0d0d0d0,0,0x01,5,02:d1:d1:d1:d1:d1,0,0x03,1,02:d2:d2:d2:d2:d2 >> $RFSERVERINTERNAL + #echo 0x2d0d0d0d0d0,0,0x03,2,02:d1:d1:d1:d1:d1,0,0x02,5,02:d2:d2:d2:d2:d2 >> $RFSERVERINTERNAL + fi + + echo_bold "-> Starting the management network ($RFBR)..." + add_local_br $RFBR + ifconfig $RFBR $HOSTVMIP + + echo_bold "-> Starting RFServer..." + #winpdb ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + ./rfserver/rfserver.py $RFSERVERCONFIG -i $RFSERVERINTERNAL -m $MULTITABLEDPS -s $SATELLITEDPS & + + echo_bold "-> Starting the controller ($ACTION) and RFPRoxy..." + case "$ACTION" in + RYU) + cd $RYU_HOME + ryu-manager --use-stderr --ofp-tcp-listen-port=$CONTROLLER_PORT ryu-rfproxy/rfproxy.py & + ;; + esac + cd - &> /dev/null + wait_port_listen $CONTROLLER_PORT + check_local_br_up tcp:127.0.0.1:$CONTROLLER_PORT + + echo_bold "-> Starting the control plane network ($RFDP VS)..." + $VSCTL add-br $RFDP + $VSCTL set bridge $RFDP other-config:datapath-id=$RFDPID + $VSCTL set bridge $RFDP protocols=$OFP + $VSCTL set-controller $RFDP tcp:127.0.0.1:$CONTROLLER_PORT + $OFCTL add-flow $RFDP actions=CONTROLLER:65509 + ifconfig $RFDP up + check_local_br_up $RFDP + + echo_bold "-> Waiting for $RFDP to connect to controller..." + while ! $VSCTL find Controller target=\"tcp:127.0.0.1:$CONTROLLER_PORT\" is_connected=true | grep -q connected ; do + echo -n . + sleep 1 + done + + echo_bold "-> Starting virtual machines..." + start_rfvms + while ! ifconfig -s rfvmA.0 ; do + echo -n . + sleep 1 + done + + # Add VM eth0 port to management bridge + for vm in rfvmA rfvmB; do + $VSCTL add-port $RFBR $vm.0 + done + + # Add VM interfaces to dataplane bridge + for i in `netstat -i|grep rfvm|cut -f 1 -d " "` ; do + if [ "$i" != "`echo $i | grep .0`" ] ; then + $VSCTL add-port $RFDP $i + fi + done + + echo_bold "-> Waiting for VMs to come up..." + while ! ping -W 1 -c 1 192.168.10.101 ; do + echo -n . + sleep 1 + done + + echo_bold "You can stop this test by pressing Ctrl+C." + wait +fi +exit 0 \ No newline at end of file diff --git a/mytest/dynamic-test3.py b/mytest/dynamic-test3.py new file mode 100755 index 00000000..8c4fab5f --- /dev/null +++ b/mytest/dynamic-test3.py @@ -0,0 +1,156 @@ +#!/usr/bin/python +import sys +from mininet.net import Mininet +from mininet.node import Controller, RemoteController +from mininet.node import CPULimitedHost, Host, Node +from mininet.node import OVSSwitch +from mininet.cli import CLI +from mininet.log import setLogLevel, info +from mininet.link import TCLink +from mininet.util import run + +import time +import xmlrpclib +import json + +# +# This will create two switches with one inter-link +# It takes input argument as the number of host ports for each sw +# the topology will look like +# _____ _____ +# | | | | +# | VMA | | VMB | +# |_____| |_____| +# | | +# __|_ __|_ +# | |(2)-----1Mb,10ms------(2)| | +# h1----(1)| s1 |(3)----10Mb,10ms------(3)| s2 |(1)-----h2 +# |____|(4)----20Mb,10ms------(4)|____| +# +# +def myNetwork(): + + rfserver_ip = '192.168.56.101' + # Initialize RPC connection to RFServer. + # + info("Connect to RFServer...\n") + rfserver = xmlrpclib.ServerProxy('http://' + '192.168.56.101' +':8008', + allow_none=True) + + info("Prepare network configuration for the tests...\n") + net = Mininet(topo=None, build=False, link=TCLink, switch=OVSSwitch) + info('---------- Add controller ----------\n') + c0 = net.addController(name='c0', controller=RemoteController, + ip=rfserver_ip, port=6633) + + info('---------- Add switches\n ----------') + s1 = net.addSwitch(name='s1', protocols='OpenFlow13') + s2 = net.addSwitch(name='s2', protocols='OpenFlow13') + #s3 = net.addSwitch(name='s3', protocols='OpenFlow13') + + info('*** Add hosts\n') + h1 = net.addHost(name="h1", cls=Host, ip='172.16.1.2/24', defaultRoute='via 172.16.1.1') + h2 = net.addHost(name="h2", cls=Host, ip='172.16.2.2/24', defaultRoute='via 172.16.2.1') + + info("---------- Add links ----------\n") + net.addLink(h1, s1) + net.addLink(h2, s2) + + net.addLink(s1, s2, bw=1, delay='10ms') + net.addLink(s1, s2, bw=10, delay='10ms') + net.addLink(s1, s2, bw=20, delay='10ms') + #net.addLink(s1, s3) + #net.addLink(s2, s3) + + info('---------- Start network ----------\n') + net.build() + net.start() + run("ovs-vsctl set bridge s1 protocols=OpenFlow13") + run("ovs-vsctl set bridge s2 protocols=OpenFlow13") + run("ovs-vsctl set bridge s3 protocols=OpenFlow13") + + info('\n\n') + prepare(rfserver, net) + info('Start the first test: \n') + first_test(rfserver, net) + info('\n\n') + info('Start the second test: \n') + second_test(rfserver, net) + info('\n\n') + info('Start the third test: \n') + third_test(rfserver, net) + try: + while True: + pass + except KeyboardInterrupt: + pass + #CLI(net) + net.stop() + +def prepare(rfserver, net): + config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 1, 'dp_id': '1', 'dp_port': 1}, + {'vm_id': '2b0b0b0b0b0', 'vm_port': 1, 'dp_id': '2', 'dp_port': 1}] + + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + +def first_test(rfserver, net): + info('Test 1 starting...\n') + info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,2') + config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '1', 'dp_port': 2}, + {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 2}] + + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + + net.get('h1').cmd('ping -c 1 172.16.1.1') + net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') + time.sleep(10) + info(net.iperf()) + info('\n') + rfserver.delete_map_configs(config[0]['vm_id']) + info('Test completed') + +def second_test(rfserver, net): + info('Test 1 starting...\n') + info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,3') + config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '1', 'dp_port': 3}, + {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 3}] + + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + + net.get('h1').cmd('ping -c 1 172.16.1.1') + net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') + time.sleep(10) + info(net.iperf()) + info('\n') + rfserver.delete_map_configs(config[0]['vm_id']) + info('Test completed\n') + +def third_test(rfserver, net): + info('Test 1 starting...\n') + info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,4') + config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '1', 'dp_port': 4}, + {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 4}] + + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + 0, entry['dp_id'], entry['dp_port']) + + net.get('h1').cmd('ping -c 1 172.16.1.1') + net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') + time.sleep(10) + info(net.iperf()) + info('\n') + rfserver.delete_map_configs(config[0]['vm_id']) + info('Test completed\n') + +if __name__ == '__main__': + setLogLevel('info') + myNetwork() \ No newline at end of file From e60c9a2721cbe73f6da6f683438b4feb480631ce Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 21:47:27 +1300 Subject: [PATCH 30/52] fix test script Signed-off-by: Trung Truong --- dynamic-test3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamic-test3.sh b/dynamic-test3.sh index 12b40f84..7f045284 100755 --- a/dynamic-test3.sh +++ b/dynamic-test3.sh @@ -133,7 +133,7 @@ auto eth2 iface eth2 inet static address 10.0.0.$i netmask 255.255.255.0 -up route add -net 172.16.$i.0 netmask 255.255.255.0 gw 10.0.0.$j +up route add -net 172.16.$j.0 netmask 255.255.255.0 gw 10.0.0.$j EOF i=$(($i+1)) j=$(($j-1)) From 802465f6be775d9a5e8ed4ed3c0e2857b9a3f0da Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 21:51:39 +1300 Subject: [PATCH 31/52] fix test script Signed-off-by: Trung Truong --- mytest/dynamic-test3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mytest/dynamic-test3.py b/mytest/dynamic-test3.py index 8c4fab5f..77fbdad9 100755 --- a/mytest/dynamic-test3.py +++ b/mytest/dynamic-test3.py @@ -30,11 +30,11 @@ # def myNetwork(): - rfserver_ip = '192.168.56.101' + rfserver_ip = '192.168.56.111' # Initialize RPC connection to RFServer. # info("Connect to RFServer...\n") - rfserver = xmlrpclib.ServerProxy('http://' + '192.168.56.101' +':8008', + rfserver = xmlrpclib.ServerProxy('http://' + rfserver_ip +':8008', allow_none=True) info("Prepare network configuration for the tests...\n") From 69355eb0b42b4203716edfadf4b3eb62f202ecf0 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 21:54:04 +1300 Subject: [PATCH 32/52] again, fixing test script, hah Signed-off-by: Trung Truong --- mytest/dynamic-test3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mytest/dynamic-test3.py b/mytest/dynamic-test3.py index 77fbdad9..3e3ae8e0 100755 --- a/mytest/dynamic-test3.py +++ b/mytest/dynamic-test3.py @@ -91,7 +91,8 @@ def prepare(rfserver, net): config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 1, 'dp_id': '1', 'dp_port': 1}, {'vm_id': '2b0b0b0b0b0', 'vm_port': 1, 'dp_id': '2', 'dp_port': 1}] - rfserver.add_map_config(entry['vm_id'], entry['vm_port'], + for entry in config: + rfserver.add_map_config(entry['vm_id'], entry['vm_port'], 0, entry['dp_id'], entry['dp_port']) def first_test(rfserver, net): From 6550533af1d254d86990095408d752ae5af394ab Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 22:16:32 +1300 Subject: [PATCH 33/52] remove checking if a dp has registered in rftable before creating a routemodtranslator for it. This causes trouble when creating mapping config after a dp has started Signed-off-by: Trung Truong --- rfserver/rfserver.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index c967cfea..213ba2bc 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -1002,20 +1002,20 @@ def config_dp(self, ct_id, dp_id): # will false, the subsequences are OK, then we create RM translator # I'm going to change this, so it can work when run RF with empty # config??? - if (self.rftable.is_dp_registered(ct_id, dp_id) or - self.isltable.is_dp_registered(ct_id, dp_id)): - if dp_id not in self.route_mod_translator: - self.log.info("Configuring datapath (dp_id=%s)" % format_id(dp_id)) - if dp_id in self.multitabledps: - self.route_mod_translator[dp_id] = NoviFlowMultitableRouteModTranslator( - dp_id, ct_id, self.rftable, self.isltable, self.log) - elif dp_id in self.satellitedps: - self.route_mod_translator[dp_id] = SatelliteRouteModTranslator( - dp_id, ct_id, self.rftable, self.isltable, self.log) - else: - self.route_mod_translator[dp_id] = DefaultRouteModTranslator( - dp_id, ct_id, self.rftable, self.isltable, self.log) - self.send_datapath_config_messages(ct_id, dp_id) + #if (self.rftable.is_dp_registered(ct_id, dp_id) or + # self.isltable.is_dp_registered(ct_id, dp_id)): + if dp_id not in self.route_mod_translator: + self.log.info("Configuring datapath (dp_id=%s)" % format_id(dp_id)) + if dp_id in self.multitabledps: + self.route_mod_translator[dp_id] = NoviFlowMultitableRouteModTranslator( + dp_id, ct_id, self.rftable, self.isltable, self.log) + elif dp_id in self.satellitedps: + self.route_mod_translator[dp_id] = SatelliteRouteModTranslator( + dp_id, ct_id, self.rftable, self.isltable, self.log) + else: + self.route_mod_translator[dp_id] = DefaultRouteModTranslator( + dp_id, ct_id, self.rftable, self.isltable, self.log) + self.send_datapath_config_messages(ct_id, dp_id) return False # DatapathDown methods def set_dp_down(self, ct_id, dp_id): From 2a5443d87fdc3df96ed2e4a4c58aa2b5b3dbf0d3 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sat, 10 Jan 2015 22:26:34 +1300 Subject: [PATCH 34/52] changes in test script Signed-off-by: Trung Truong --- mytest/dynamic-test3.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/mytest/dynamic-test3.py b/mytest/dynamic-test3.py index 3e3ae8e0..20933220 100755 --- a/mytest/dynamic-test3.py +++ b/mytest/dynamic-test3.py @@ -67,24 +67,19 @@ def myNetwork(): net.start() run("ovs-vsctl set bridge s1 protocols=OpenFlow13") run("ovs-vsctl set bridge s2 protocols=OpenFlow13") - run("ovs-vsctl set bridge s3 protocols=OpenFlow13") + #run("ovs-vsctl set bridge s3 protocols=OpenFlow13") info('\n\n') prepare(rfserver, net) - info('Start the first test: \n') + #info('Start the first test: \n') first_test(rfserver, net) info('\n\n') info('Start the second test: \n') - second_test(rfserver, net) + #second_test(rfserver, net) info('\n\n') info('Start the third test: \n') - third_test(rfserver, net) - try: - while True: - pass - except KeyboardInterrupt: - pass - #CLI(net) + #third_test(rfserver, net) + CLI(net) net.stop() def prepare(rfserver, net): @@ -97,7 +92,7 @@ def prepare(rfserver, net): def first_test(rfserver, net): info('Test 1 starting...\n') - info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,2') + info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,2\n') config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '1', 'dp_port': 2}, {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 2}] @@ -109,14 +104,15 @@ def first_test(rfserver, net): net.get('h2').cmd('ping -c 1 172.16.2.1') info('Wait for flow entries installation completed...\n') time.sleep(10) - info(net.iperf()) + #info(net.iperf()) info('\n') - rfserver.delete_map_configs(config[0]['vm_id']) + #rfserver.delete_map_configs(config[0]['vm_id']) + #rfserver.delete_map_configs(config[1]['vm_id']) info('Test completed') def second_test(rfserver, net): info('Test 1 starting...\n') - info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,3') + info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,3\n') config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '1', 'dp_port': 3}, {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 3}] @@ -131,11 +127,12 @@ def second_test(rfserver, net): info(net.iperf()) info('\n') rfserver.delete_map_configs(config[0]['vm_id']) + rfserver.delete_map_configs(config[1]['vm_id']) info('Test completed\n') def third_test(rfserver, net): info('Test 1 starting...\n') - info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,4') + info('Mapping configuration: rfvmA, rfvmB port 1,2 are mapped to S1, S2 port 1,4\n') config = [{'vm_id': '2a0a0a0a0a0', 'vm_port': 2, 'dp_id': '1', 'dp_port': 4}, {'vm_id': '2b0b0b0b0b0', 'vm_port': 2, 'dp_id': '2', 'dp_port': 4}] @@ -150,6 +147,7 @@ def third_test(rfserver, net): info(net.iperf()) info('\n') rfserver.delete_map_configs(config[0]['vm_id']) + rfserver.delete_map_configs(config[1]['vm_id']) info('Test completed\n') if __name__ == '__main__': From ecacfdc9d17f7cfc0be3155957b888e7ecba8d03 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Sun, 11 Jan 2015 07:14:40 +1300 Subject: [PATCH 35/52] Change dynamic-test3 script to use ospf for LXCs --- dynamic-test3.sh | 52 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/dynamic-test3.sh b/dynamic-test3.sh index 7f045284..3bc190c8 100755 --- a/dynamic-test3.sh +++ b/dynamic-test3.sh @@ -111,9 +111,44 @@ start_ovs() { start_rfvms() { i=1 - j=2 for vm in rfvmA rfvmB; do ROOTFS=$LXCDIR/$vm/rootfs + + # Quagga ospf configuration + cat > $ROOTFS/etc/quagga/ospfd.conf < $ROOTFS/etc/quagga/zebra.conf < $ROOTFS/etc/network/interfaces <> $ROOTFS/etc/network/interfaces< $ROOTFS/etc/rc.local < Date: Mon, 12 Jan 2015 22:40:07 +1300 Subject: [PATCH 36/52] Fix the issue with flow table update. When changing mapping config, we have to wait until ARP expires to have switch table updated. This makes rfclient to send RouteMods as rfserver's request Signed-off-by: Trung Truong --- mytest/dynamic-test3.py | 6 +++--- rfclient/FlowTable.cc | 16 ++++++++++++++++ rfclient/FlowTable.hh | 6 ++++++ rfclient/RFClient.cc | 4 ++++ rflib/defs.py | 3 ++- rfserver/rfserver.py | 2 -- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/mytest/dynamic-test3.py b/mytest/dynamic-test3.py index 20933220..d0c52102 100755 --- a/mytest/dynamic-test3.py +++ b/mytest/dynamic-test3.py @@ -56,9 +56,9 @@ def myNetwork(): net.addLink(h1, s1) net.addLink(h2, s2) - net.addLink(s1, s2, bw=1, delay='10ms') - net.addLink(s1, s2, bw=10, delay='10ms') - net.addLink(s1, s2, bw=20, delay='10ms') + net.addLink(s1, s2, bw=1, delay='1ms') + net.addLink(s1, s2, bw=10, delay='2ms') + net.addLink(s1, s2, bw=20, delay='4ms') #net.addLink(s1, s3) #net.addLink(s2, s3) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 7bf67f4e..6df45ff0 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -707,3 +707,19 @@ int FlowTable::sendToHw(RouteModType mod, const IPAddress& addr, this->sendRm(rm); return 0; } + +void FlowTable::flushRouteMod(Interface& iface) { + syslog(LOG_INFO, "debug: Interface %s", iface.toString()); + map::iterator h_it; + for (h_it = this->hostTable.begin(); h_it != this->hostTable.end(); h_it++) { + HostEntry& hentry = h_it->second; + if (hentry.interface == iface) + this->sendToHw(RMT_ADD, hentry); + } + map::iterator r_it; + for (r_it = this->routeTable.begin(); r_it != this->routeTable.end(); ++r_it) { + RouteEntry& rentry = r_it->second; + if (rentry.interface == iface) + this->sendToHw(RMT_ADD, r_it->second); + } +} diff --git a/rfclient/FlowTable.hh b/rfclient/FlowTable.hh index 2f9894c7..5acdf3c8 100644 --- a/rfclient/FlowTable.hh +++ b/rfclient/FlowTable.hh @@ -74,6 +74,12 @@ class FlowTable { void updateRouteTable(struct rtnl_route *route, int action); uint64_t get_vm_id(); + /* This function read HostEntry and RouteEntry entries from HostTable + * & RouteTable, create RouteMod msgs and flush to rfserver. + * It will be invoked when RFClient receives a request from rfserver. + * It is part of dynamic routeflow. */ + void flushRouteMod(Interface& iface); + private: RouteSource source; InterfaceMap* ifMap; diff --git a/rfclient/RFClient.cc b/rfclient/RFClient.cc index ccdb720f..8c7c6559 100644 --- a/rfclient/RFClient.cc +++ b/rfclient/RFClient.cc @@ -259,6 +259,10 @@ void RFClient::sendAllInterfaceToControllerRouteMods(uint32_t vm_port) { if (iface.port == vm_port) { iface.active = true; sendInterfaceToControllerRouteMods(iface); + /* We're gonna to send all available entries from the host & route table + * as well. That will update hardware table without waiting for ARP request + * from hosts */ + this->flowTable->flushRouteMod(iface); } } } diff --git a/rflib/defs.py b/rflib/defs.py index b3125046..c9ca5403 100644 --- a/rflib/defs.py +++ b/rflib/defs.py @@ -63,7 +63,8 @@ PCT_RESET = 1 # Reset the client port to inactive. PCT_MAP_SUCCESS = 2 # Mapping was successful; port can be brought up. PCT_ROUTEMOD_ACK = 3 - +PCT_FLUSH_REQ = 4 # Request the client to send RouteMod from HostTable & RouteTable + # It is used to update flow table when mapping config changes PC_MAP = 0 PC_RESET = 1 diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 213ba2bc..69a58351 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -666,8 +666,6 @@ def add_map_config(self, vm_id, vm_port, ct_id, dp_id, dp_port): (format_id(rf_entry.dp_id), rf_entry.dp_port, format_id(rf_entry.vs_id), rf_entry.vs_port)) - - return True # Update an existing mapping: From 898ee09fe98743fb2c6c9536bd34b287f4eff4ab Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Mon, 12 Jan 2015 22:48:25 +1300 Subject: [PATCH 37/52] Fix minor error with printing string Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 6df45ff0..bb3c6d4c 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -709,7 +709,7 @@ int FlowTable::sendToHw(RouteModType mod, const IPAddress& addr, } void FlowTable::flushRouteMod(Interface& iface) { - syslog(LOG_INFO, "debug: Interface %s", iface.toString()); + syslog(LOG_INFO, "debug: Interface %s", iface.toString().c_str()); map::iterator h_it; for (h_it = this->hostTable.begin(); h_it != this->hostTable.end(); h_it++) { HostEntry& hentry = h_it->second; From 9e22a9bbfe1c6f3876fa2697527af3e2dfdad6de Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Mon, 12 Jan 2015 23:03:45 +1300 Subject: [PATCH 38/52] Small changes in dynamic test 3 Signed-off-by: Trung Truong --- dynamic-test3.sh | 21 ++++++++++----------- mytest/dynamic-test3.py | 16 +++++++--------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/dynamic-test3.sh b/dynamic-test3.sh index 3bc190c8..0b5d99b4 100755 --- a/dynamic-test3.sh +++ b/dynamic-test3.sh @@ -111,6 +111,7 @@ start_ovs() { start_rfvms() { i=1 + j=2 for vm in rfvmA rfvmB; do ROOTFS=$LXCDIR/$vm/rootfs @@ -120,8 +121,8 @@ password 123 enable password 123 ! router ospf - network 172.16.0.0/12 area 0 - network 10.0.0.0/8 area 0 + network 172.16.0.0/12 area 0 + network 10.0.0.0/8 area 0 passive-interface eth1 log file /var/log/quagga/ospfd.log @@ -131,8 +132,8 @@ interface eth1 ip ospf dead-interval 4 interface eth2 - ip ospf hello-interval 1 - ip ospf dead-interval 4 + ip ospf hello-interval 1 + ip ospf dead-interval 4 ! EOF cat > $ROOTFS/etc/quagga/zebra.conf < $ROOTFS/etc/network/interfaces < $ROOTFS/etc/rc.local < $RFSERVERINTERNAL - #echo 0x2a0a0a0a0a0,0,0x01,2,02:a1:a1:a1:a1:a1,0,0x02,2,02:a2:a2:a2:a2:a2 >> $RFSERVERINTERNAL - + #echo 0x2a0a0a0a0a0,0,0x01,2,02:a1:a1:a1:a1:a1,0,0x02,2,02:a2:a2:a2:a2:a2 >> $RFSERVERINTERNAL #echo 0x2b0b0b0b0b0,0,0x01,3,02:b1:b1:b1:b1:b1,0,0x02,3,02:b2:b2:b2:b2:b2 >> $RFSERVERINTERNAL - #echo 0x2c0c0c0c0c0,0,0x01,4,02:c1:c1:c1:c1:c1,0,0x02,4,02:c2:c2:c2:c2:c2 >> $RFSERVERINTERNAL - #echo 0x2d0d0d0d0d0,0,0x01,5,02:d1:d1:d1:d1:d1,0,0x03,1,02:d2:d2:d2:d2:d2 >> $RFSERVERINTERNAL #echo 0x2d0d0d0d0d0,0,0x03,2,02:d1:d1:d1:d1:d1,0,0x02,5,02:d2:d2:d2:d2:d2 >> $RFSERVERINTERNAL fi diff --git a/mytest/dynamic-test3.py b/mytest/dynamic-test3.py index d0c52102..1992dd2b 100755 --- a/mytest/dynamic-test3.py +++ b/mytest/dynamic-test3.py @@ -99,15 +99,17 @@ def first_test(rfserver, net): for entry in config: rfserver.add_map_config(entry['vm_id'], entry['vm_port'], 0, entry['dp_id'], entry['dp_port']) - + # The ping cmd is used to generate ARP request. We need flow entries to exist before + # iperf runs net.get('h1').cmd('ping -c 1 172.16.1.1') net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') time.sleep(10) - #info(net.iperf()) + info(net.iperf()) info('\n') - #rfserver.delete_map_configs(config[0]['vm_id']) - #rfserver.delete_map_configs(config[1]['vm_id']) + rfserver.delete_map_configs(config[0]['vm_id']) + rfserver.delete_map_configs(config[1]['vm_id']) info('Test completed') def second_test(rfserver, net): @@ -120,8 +122,6 @@ def second_test(rfserver, net): rfserver.add_map_config(entry['vm_id'], entry['vm_port'], 0, entry['dp_id'], entry['dp_port']) - net.get('h1').cmd('ping -c 1 172.16.1.1') - net.get('h2').cmd('ping -c 1 172.16.2.1') info('Wait for flow entries installation completed...\n') time.sleep(10) info(net.iperf()) @@ -139,9 +139,7 @@ def third_test(rfserver, net): for entry in config: rfserver.add_map_config(entry['vm_id'], entry['vm_port'], 0, entry['dp_id'], entry['dp_port']) - - net.get('h1').cmd('ping -c 1 172.16.1.1') - net.get('h2').cmd('ping -c 1 172.16.2.1') + info('Wait for flow entries installation completed...\n') time.sleep(10) info(net.iperf()) From 36a83ea916a1bfff06ba1ea7f0c2ca1b2e988783 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 00:04:52 +1300 Subject: [PATCH 39/52] adding log to debug rfclient Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index bb3c6d4c..e36b9209 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -713,13 +713,22 @@ void FlowTable::flushRouteMod(Interface& iface) { map::iterator h_it; for (h_it = this->hostTable.begin(); h_it != this->hostTable.end(); h_it++) { HostEntry& hentry = h_it->second; + syslog(LOG_INFO, "debug: read a hostentry Ip:%s, Mac:%s, int:%s", + hentry.address.toString().c_str(), + hentry.hwaddress.toString().c_str(), + hentry.interface.toString().c_str()); if (hentry.interface == iface) this->sendToHw(RMT_ADD, hentry); } map::iterator r_it; for (r_it = this->routeTable.begin(); r_it != this->routeTable.end(); ++r_it) { RouteEntry& rentry = r_it->second; + syslog(LOG_INFO, "debug: read a routeentry Ip:%s, Mac:%s, int:%s", + rentry.address.toString().c_str(), + rentry.hwaddress.toString().c_str(), + rentry.interface.toString().c_str()); if (rentry.interface == iface) + syslog(LOG_INFO, "debug: send a routemod"); this->sendToHw(RMT_ADD, r_it->second); } } From 7dcaf17635b074e322338672e0fe7610168fd80a Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 00:07:21 +1300 Subject: [PATCH 40/52] fix calling wrong RouteEntry member Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index e36b9209..4a928a65 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -725,7 +725,7 @@ void FlowTable::flushRouteMod(Interface& iface) { RouteEntry& rentry = r_it->second; syslog(LOG_INFO, "debug: read a routeentry Ip:%s, Mac:%s, int:%s", rentry.address.toString().c_str(), - rentry.hwaddress.toString().c_str(), + rentry.gateway.toString().c_str(), rentry.interface.toString().c_str()); if (rentry.interface == iface) syslog(LOG_INFO, "debug: send a routemod"); From f233fa3693505b3fa318ff997f3cda2aea93c074 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 15:01:20 +1300 Subject: [PATCH 41/52] Fix the hostTabe zero entry when trying to read from RFClient thread. Need to pass by reference not by value to FlowTable thread --- dynamic-test3.sh | 7 +++++-- rfclient/FlowTable.cc | 7 ++++++- rfclient/RFClient.cc | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dynamic-test3.sh b/dynamic-test3.sh index 0b5d99b4..4ac765c0 100755 --- a/dynamic-test3.sh +++ b/dynamic-test3.sh @@ -192,6 +192,7 @@ stop_rfvms() { lxc-stop -n $vm &> /dev/null; ROOTFS=$LXCDIR/$vm/rootfs rm -rf $ROOTFS/var/run/network/ifstate; + cp /dev/null $ROOTFS/var/log/syslog done } @@ -227,8 +228,10 @@ if [ "$ACTION" != "RESET" ]; then # Initially, rfvmA is mapped the port1 to dp1, port1 # & port2 to dp2, port1 - #echo 0x2a0a0a0a0a0,1,0,0x01,1 >> $RFSERVERCONFIG - #echo 0x2a0a0a0a0a0,2,0,0x02,1 >> $RFSERVERCONFIG + echo 0x2a0a0a0a0a0,1,0,0x01,1 >> $RFSERVERCONFIG + echo 0x2a0a0a0a0a0,2,0,0x01,2 >> $RFSERVERCONFIG + echo 0x2b0b0b0b0b0,1,0,0x02,1 >> $RFSERVERCONFIG + echo 0x2b0b0b0b0b0,2,0,0x02,2 >> $RFSERVERCONFIG cp /dev/null $RFSERVERINTERNAL echo "vm_id,ct_id,dp_id,dp_port,eth_addr,rem_ct,rem_id,rem_port,rem_eth_addr" > $RFSERVERINTERNAL diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 4a928a65..2938f810 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -124,6 +124,8 @@ void FlowTable::initNLListener() { syslog(LOG_NOTICE, "Netlink interface enabled"); sock = nl_socket_alloc(); + syslog(LOG_INFO, "Inside flowtable class addr=%p", this); + err = nl_cache_mngr_alloc(sock, NETLINK_ROUTE, 0, &mngr); if (err < 0) { throw "some kind of problem with the route cache manager!"; @@ -391,6 +393,7 @@ void FlowTable::updateHostTable(struct rtnl_neigh *neigh, // Add to host table boost::lock_guard lock(hostTableMutex); this->hostTable[host] = *hentry; + syslog(LOG_INFO, "HostTable size %d, addr=%p", this->hostTable.size(), &this->hostTable); } // If we have been attempting neighbour discovery for this // host, then we can close the associated socket. @@ -709,9 +712,11 @@ int FlowTable::sendToHw(RouteModType mod, const IPAddress& addr, } void FlowTable::flushRouteMod(Interface& iface) { + boost::lock_guard lock(hostTableMutex); + syslog(LOG_INFO, "debug: HostTable size %d, ht_addr=%p, ft_add=%p", hostTable.size(), &hostTable, this); syslog(LOG_INFO, "debug: Interface %s", iface.toString().c_str()); map::iterator h_it; - for (h_it = this->hostTable.begin(); h_it != this->hostTable.end(); h_it++) { + for (h_it = hostTable.begin(); h_it != hostTable.end(); h_it++) { HostEntry& hentry = h_it->second; syslog(LOG_INFO, "debug: read a hostentry Ip:%s, Mac:%s, int:%s", hentry.address.toString().c_str(), diff --git a/rfclient/RFClient.cc b/rfclient/RFClient.cc index 8c7c6559..9b444653 100644 --- a/rfclient/RFClient.cc +++ b/rfclient/RFClient.cc @@ -157,7 +157,7 @@ RFClient::RFClient(uint64_t id, const string &address, RouteSource source) { void RFClient::startFlowTable(RouteSource source) { this->flowTable = new FlowTable(this->id, this, &(this->rm_q), source); - boost::thread t(*this->flowTable); + boost::thread t(boost::ref(*this->flowTable)); t.detach(); } From ca014216a507a24c4c4fcc9011fffc07c0d84067 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 16:01:01 +1300 Subject: [PATCH 42/52] Some wording changes Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 23 ++++++++++++----------- rfserver/rfserver.py | 2 -- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 2938f810..f95689fd 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -718,22 +718,23 @@ void FlowTable::flushRouteMod(Interface& iface) { map::iterator h_it; for (h_it = hostTable.begin(); h_it != hostTable.end(); h_it++) { HostEntry& hentry = h_it->second; - syslog(LOG_INFO, "debug: read a hostentry Ip:%s, Mac:%s, int:%s", - hentry.address.toString().c_str(), - hentry.hwaddress.toString().c_str(), - hentry.interface.toString().c_str()); - if (hentry.interface == iface) + if (hentry.interface == iface) { this->sendToHw(RMT_ADD, hentry); + syslog(LOG_INFO, "debug: read a hostentry Ip:%s, Mac:%s, int:%s", + hentry.address.toString().c_str(), + hentry.hwaddress.toString().c_str(), + hentry.interface.toString().c_str()); + } } map::iterator r_it; for (r_it = this->routeTable.begin(); r_it != this->routeTable.end(); ++r_it) { RouteEntry& rentry = r_it->second; - syslog(LOG_INFO, "debug: read a routeentry Ip:%s, Mac:%s, int:%s", - rentry.address.toString().c_str(), - rentry.gateway.toString().c_str(), - rentry.interface.toString().c_str()); - if (rentry.interface == iface) - syslog(LOG_INFO, "debug: send a routemod"); + if (rentry.interface == iface) { this->sendToHw(RMT_ADD, r_it->second); + syslog(LOG_INFO, "debug: read a routeentry Ip:%s, Mac:%s, int:%s", + rentry.address.toString().c_str(), + rentry.gateway.toString().c_str(), + rentry.interface.toString().c_str()); + } } } diff --git a/rfserver/rfserver.py b/rfserver/rfserver.py index 69a58351..28920fee 100755 --- a/rfserver/rfserver.py +++ b/rfserver/rfserver.py @@ -1061,14 +1061,12 @@ def map_port(self, vm_id, vm_port, vs_id, vs_port): # Disable this code portion for now, because it causes an issue with dp # comming back from a reset (when restart Mininet, no flow entries are # installed - """ entry = self.vmporttable.get_vm_port_info(vm_id = vm_id, vm_port = vm_port) if entry is not None: entry.update_vs(vs_id=vs_id, vs_port=vs_port) self.vmporttable.set_entry(entry) #TODO: Because we keep VS info, so no need RFClient to send this msg anymore # When register dp port, we can get vm and vs info to update rftable - """ entry = None entry = self.rftable.get_entry_by_vm_port(vm_id, vm_port) if entry is not None and entry.get_status() == RFENTRY_ASSOCIATED: From caa7fb44e73e09f030182ddd971a54d1552e9dc9 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 16:17:33 +1300 Subject: [PATCH 43/52] Fix RFServer CLI error when trying to format vm_id Signed-off-by: Trung Truong --- rfserver/rfservercli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py index 9d9c41e1..d25b481a 100755 --- a/rfserver/rfservercli.py +++ b/rfserver/rfservercli.py @@ -16,7 +16,7 @@ def format_id(value): try: value = int(value) return defs.format_id(value) - except ValueError, TypeError: + except: return value class DeleteCommand(Command): From 57b9784ab2c4f811638742f64c6e3d0b1ff80d09 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 17:10:27 +1300 Subject: [PATCH 44/52] add some print for debugging Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index f95689fd..9945432c 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -218,6 +218,7 @@ void FlowTable::GWResolverCb(FlowTable *ft) { addr_str.c_str()); } else { ft->routeTable.insert(make_pair(re_key, re)); + syslog(LOG_INFO, "debug: RouteTable size %d", ft->routeTable.size()); if (ft->findHost(re.gateway) == MAC_ADDR_NONE) { syslog(LOG_ERR, "Cannot resolve gateway %s, will retry route %s/%s", @@ -713,7 +714,7 @@ int FlowTable::sendToHw(RouteModType mod, const IPAddress& addr, void FlowTable::flushRouteMod(Interface& iface) { boost::lock_guard lock(hostTableMutex); - syslog(LOG_INFO, "debug: HostTable size %d, ht_addr=%p, ft_add=%p", hostTable.size(), &hostTable, this); + syslog(LOG_INFO, "debug: HostTable size %d", hostTable.size()); syslog(LOG_INFO, "debug: Interface %s", iface.toString().c_str()); map::iterator h_it; for (h_it = hostTable.begin(); h_it != hostTable.end(); h_it++) { @@ -727,10 +728,11 @@ void FlowTable::flushRouteMod(Interface& iface) { } } map::iterator r_it; - for (r_it = this->routeTable.begin(); r_it != this->routeTable.end(); ++r_it) { + syslog(LOG_INFO, "debug: RouteTable size %d", routeTable.size()); + for (r_it = routeTable.begin(); r_it != routeTable.end(); r_it++) { RouteEntry& rentry = r_it->second; if (rentry.interface == iface) { - this->sendToHw(RMT_ADD, r_it->second); + this->sendToHw(RMT_ADD, rentry); syslog(LOG_INFO, "debug: read a routeentry Ip:%s, Mac:%s, int:%s", rentry.address.toString().c_str(), rentry.gateway.toString().c_str(), From 549c22be24cab7fd6a90ab9970a838b5b51dc71c Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Tue, 13 Jan 2015 17:25:18 +1300 Subject: [PATCH 45/52] Switch LOG_DEBUG to LOG_INFO to see if messages appear on syslog Signed-off-by: Trung Truong --- rfclient/FlowTable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rfclient/FlowTable.cc b/rfclient/FlowTable.cc index 9945432c..651c0110 100644 --- a/rfclient/FlowTable.cc +++ b/rfclient/FlowTable.cc @@ -589,7 +589,7 @@ int FlowTable::resolveGateway(const IPAddress& gateway, } // Otherwise, we should go ahead and begin the process. - syslog(LOG_DEBUG, "starting neighbour discovery for %s", gateway_str.c_str()); + syslog(LOG_INFO, "starting neighbour discovery for %s", gateway_str.c_str()); int sock = initiateND(gateway_str.c_str()); if (sock == -1) { return -1; From 8f22bd0856bd0d38bff548a738b8d42dbd0102b9 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 13:15:57 +1300 Subject: [PATCH 46/52] Trying to fix the issue with "Cannot send RouteMod to down port" (because the routes were added before the port is mapped) Signed-off-by: Trung Truong --- rfclient/RFClient.cc | 6 +++++- rfserver/rfservercli.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rfclient/RFClient.cc b/rfclient/RFClient.cc index 9b444653..bc8ff44f 100644 --- a/rfclient/RFClient.cc +++ b/rfclient/RFClient.cc @@ -94,7 +94,10 @@ bool RFClient::findInterface(const char *ifName, Interface *dst) { return false; } - *dst = it->second; + /* We should assign the pointer instead of value. So FlowTable threads will get + * the up-to-date value. Try to fix it this way */ + //*dst = it->second; + dst = &it->second; return true; } @@ -263,6 +266,7 @@ void RFClient::sendAllInterfaceToControllerRouteMods(uint32_t vm_port) { * as well. That will update hardware table without waiting for ARP request * from hosts */ this->flowTable->flushRouteMod(iface); + break; } } } diff --git a/rfserver/rfservercli.py b/rfserver/rfservercli.py index d25b481a..9d9c41e1 100755 --- a/rfserver/rfservercli.py +++ b/rfserver/rfservercli.py @@ -16,7 +16,7 @@ def format_id(value): try: value = int(value) return defs.format_id(value) - except: + except ValueError, TypeError: return value class DeleteCommand(Command): From 20ab8ef822b6c3d5ea89884dcb06484084f95a87 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 13:59:57 +1300 Subject: [PATCH 47/52] new script to start & stop RFClient Signed-off-by: Trung Truong --- mytest/test3-rfclient.sh | 241 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100755 mytest/test3-rfclient.sh diff --git a/mytest/test3-rfclient.sh b/mytest/test3-rfclient.sh new file mode 100755 index 00000000..b52d2e2d --- /dev/null +++ b/mytest/test3-rfclient.sh @@ -0,0 +1,241 @@ +#!/bin/bash +HOME=/home/ubuntu +RF_HOME=$HOME/RouteFlow +LXCDIR=/var/lib/lxc +RFBR0=br0 +RFDP0=dp0 +RFDPID=7266767372667673 + +RFSERVERIP=192.168.56.111 +RFPROXYIP=$RFSERVERIP +HOSTVMIP=192.168.10.1 +VSCTL="ovs-vsctl" +OFCTL="ovs-ofctl -O$OFP" +OFP=OpenFlow13 +CONTROLLER_PORT=6633 +ACTION="" +while getopts ":a:p:d" opt; do + case $opt in + a) p_out="$OPTARG" + ;; + p) arg_1="$OPTARG" + ;; + \?) echo "Invalid option -$OPTARG" >&2 + ;; + esac +done + +cd $RF_HOME +wait_port_listen() { + port=$1 + while ! `nc -z localhost $port` ; do + echo -n . + sleep 1 + done +} + +echo_bold() { + echo -e "\033[1m${1}\033[0m" +} + +kill_process_tree() { + top=$1 + pid=$2 + + children=`ps -o pid --no-headers --ppid ${pid}` + + for child in $children + do + kill_process_tree 0 $child + done + + if [ $top -eq 0 ]; then + kill -9 $pid &> /dev/null + fi +} + +add_local_br() { + br=$1 + dpid=$2 + $VSCTL add-br $br + $VSCTL set bridge $br protocols=$OFP + if [ "$dpid" != "" ] ; then + $VSCTL set bridge $br other-config:datapath-id=$dpid + fi + ifconfig $br up + check_local_br_up $br +} + +check_local_br_up() { + br=$1 + echo waiting for OVS sw/controller $br to come up + while ! $OFCTL ping $br 64|grep -q "64 bytes from" ; do + echo -n "." + sleep 1 + done +} + +start_ovs() { + if [ ! -f /usr/local/etc/openvswitch/conf.db ] ; then + ovsdb-tool create /usr/local/etc/openvswitch/conf.db /usr/local/share/openvswitch/vswitch.ovsschema + fi + ovsdb-server --pidfile --detach --remote=punix:$OVSSOCK + ovs-vswitchd --pidfile --detach unix:$OVSSOCK +} + +start_rfvms() { + i=1 + j=2 + for vm in rfvmA rfvmB; do + ROOTFS=$LXCDIR/$vm/rootfs + # Quagga ospf configuration + cat > $ROOTFS/etc/quagga/ospfd.conf < $ROOTFS/etc/quagga/zebra.conf < $ROOTFS/etc/network/interfaces < $ROOTFS/etc/rc.local < $ROOTFS/root/run_rfclient.sh < Stopping the virtual machines..." + for vm in rfvmA rfvmB; do + lxc-stop -n $vm &> /dev/null; + ROOTFS=$LXCDIR/$vm/rootfs + rm -rf $ROOTFS/var/run/network/ifstate; + cp /dev/null $ROOTFS/var/log/syslog + done +} + +reset() { + echo_bold "-> Stopping and resetting LXC VMs..."; + stop_rfvms + + init=$1; + if [ $init -eq 1 ]; then + echo_bold "-> Starting OVS daemons..."; + #start_ovs + else + echo_bold "-> Stopping child processes..."; + kill_process_tree 1 $$ + fi + sudo $VSCTL del-br $RFBR &> /dev/null; + sudo $VSCTL del-br $RFDP &> /dev/null; + sudo $VSCTL emer-reset &> /dev/null; +} +reset 1 +trap "reset 0; exit 0" INT + +if [ "$ACTION" == "START" ]; then + echo_bold "-> Starting the management network ($RFBR)..." + add_local_br $RFBR + ifconfig $RFBR $HOSTVMIP + + check_local_br_up tcp:$RFSERVERIP:$CONTROLLER_PORT + + echo_bold "-> Starting the control plane network ($RFDP VS)..." + $VSCTL add-br $RFDP + $VSCTL set bridge $RFDP other-config:datapath-id=$RFDPID + $VSCTL set bridge $RFDP protocols=$OFP + $VSCTL set-controller $RFDP tcp:$RFPROXYIP:$CONTROLLER_PORT + $OFCTL add-flow $RFDP actions=CONTROLLER:65509 + ifconfig $RFDP up + check_local_br_up $RFDP + + echo_bold "-> Waiting for $RFDP to connect to controller..." + while ! $VSCTL find Controller target=\"tcp:$RFPROXYIP:$CONTROLLER_PORT\" is_connected=true | grep -q connected ; do + echo -n . + sleep 1 + done + + echo_bold "-> Starting VMs..." + start_rfvms + while ! ifconfig -s rfvmA.0 ; do + echo -n . + sleep 1 + done + + # Add VM eth0 port to management bridge + for vm in rfvmA rfvmB; do + $VSCTL add-port $RFBR $vm.0 + done + + # Add VM interfaces to dataplane bridge + for i in `netstat -i|grep rfvm|cut -f 1 -d " "` ; do + if [ "$i" != "`echo $i | grep .0`" ] ; then + $VSCTL add-port $RFDP $i + fi + done + + echo_bold "-> Waiting for VMs to come up..." + while ! ping -W 1 -c 1 192.168.10.101 ; do + echo -n . + sleep 1 + done + + echo_bold "You can stop this by pressing Ctrl+C." + wait +fi +exit 0 From 1231e5250d28f87db50836b8210c151de05022dd Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 14:06:04 +1300 Subject: [PATCH 48/52] test script Signed-off-by: Trung Truong --- mytest/test3-rfclient.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/mytest/test3-rfclient.sh b/mytest/test3-rfclient.sh index b52d2e2d..1d3b19c6 100755 --- a/mytest/test3-rfclient.sh +++ b/mytest/test3-rfclient.sh @@ -13,18 +13,28 @@ VSCTL="ovs-vsctl" OFCTL="ovs-ofctl -O$OFP" OFP=OpenFlow13 CONTROLLER_PORT=6633 + +if [ "$EUID" != "0" ]; then + echo "You must be root to run this script." + exit 1 +fi + ACTION="" -while getopts ":a:p:d" opt; do - case $opt in - a) p_out="$OPTARG" +case "$1" in +--ryu) + ACTION="RYU" ;; - p) arg_1="$OPTARG" +--reset) + ACTION="RESET" ;; - \?) echo "Invalid option -$OPTARG" >&2 +*) + echo "Invalid argument: $1" + echo "Options: " + echo " --ryu: run using RYU" + echo " --reset: stop running and clear data from previous executions" + exit ;; - esac -done - +esac cd $RF_HOME wait_port_listen() { port=$1 @@ -188,7 +198,7 @@ reset() { reset 1 trap "reset 0; exit 0" INT -if [ "$ACTION" == "START" ]; then +if [ "$ACTION" == "start" ]; then echo_bold "-> Starting the management network ($RFBR)..." add_local_br $RFBR ifconfig $RFBR $HOSTVMIP From 6c663f46166fe6855cbe61e97955a8d7cdce02cf Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 14:06:04 +1300 Subject: [PATCH 49/52] test script fix Signed-off-by: Trung Truong --- mytest/test3-rfclient.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/mytest/test3-rfclient.sh b/mytest/test3-rfclient.sh index b52d2e2d..59130c6a 100755 --- a/mytest/test3-rfclient.sh +++ b/mytest/test3-rfclient.sh @@ -13,18 +13,28 @@ VSCTL="ovs-vsctl" OFCTL="ovs-ofctl -O$OFP" OFP=OpenFlow13 CONTROLLER_PORT=6633 + +if [ "$EUID" != "0" ]; then + echo "You must be root to run this script." + exit 1 +fi + ACTION="" -while getopts ":a:p:d" opt; do - case $opt in - a) p_out="$OPTARG" +case "$1" in +--ryu) + ACTION="RYU" ;; - p) arg_1="$OPTARG" +--reset) + ACTION="RESET" ;; - \?) echo "Invalid option -$OPTARG" >&2 +*) + echo "Invalid argument: $1" + echo "Options: " + echo " --ryu: run using RYU" + echo " --reset: stop running and clear data from previous executions" + exit ;; - esac -done - +esac cd $RF_HOME wait_port_listen() { port=$1 @@ -188,7 +198,7 @@ reset() { reset 1 trap "reset 0; exit 0" INT -if [ "$ACTION" == "START" ]; then +if [ "$ACTION" != "RESET" ]; then echo_bold "-> Starting the management network ($RFBR)..." add_local_br $RFBR ifconfig $RFBR $HOSTVMIP From 142e44f98d7cf6ecf2bf67f974e2eae0804537d6 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 14:16:47 +1300 Subject: [PATCH 50/52] suck the test script Signed-off-by: Trung Truong --- mytest/test3-rfclient.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mytest/test3-rfclient.sh b/mytest/test3-rfclient.sh index 59130c6a..32abe726 100755 --- a/mytest/test3-rfclient.sh +++ b/mytest/test3-rfclient.sh @@ -107,13 +107,11 @@ router ospf network 172.16.0.0/12 area 0 network 10.0.0.0/8 area 0 passive-interface eth1 - log file /var/log/quagga/ospfd.log ! interface eth1 ip ospf hello-interval 1 ip ospf dead-interval 4 - interface eth2 ip ospf hello-interval 1 ip ospf dead-interval 4 @@ -130,7 +128,6 @@ interface eth1 ! interface eth2 ip address 10.0.0.$i/24 - ip route 172.16.$j.0 255.255.255.0 10.0.0.$j EOF # Prepare configs for LXCs @@ -198,7 +195,7 @@ reset() { reset 1 trap "reset 0; exit 0" INT -if [ "$ACTION" != "RESET" ]; then +if [ "$ACTION" == "start" ]; then echo_bold "-> Starting the management network ($RFBR)..." add_local_br $RFBR ifconfig $RFBR $HOSTVMIP @@ -248,4 +245,4 @@ if [ "$ACTION" != "RESET" ]; then echo_bold "You can stop this by pressing Ctrl+C." wait fi -exit 0 +exit 0 \ No newline at end of file From 4d808657f77d3fef1f0e68a72791eef598976d5a Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 14:18:17 +1300 Subject: [PATCH 51/52] test script again Signed-off-by: Trung Truong --- mytest/test3-rfclient.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mytest/test3-rfclient.sh b/mytest/test3-rfclient.sh index fa65fe99..635aafd2 100755 --- a/mytest/test3-rfclient.sh +++ b/mytest/test3-rfclient.sh @@ -21,8 +21,8 @@ fi ACTION="" case "$1" in ---ryu) - ACTION="RYU" +--start) + ACTION="START" ;; --reset) ACTION="RESET" @@ -195,7 +195,7 @@ reset() { reset 1 trap "reset 0; exit 0" INT -if [ "$ACTION" == "start" ]; then +if [ "$ACTION" == "RESET" ]; then echo_bold "-> Starting the management network ($RFBR)..." add_local_br $RFBR ifconfig $RFBR $HOSTVMIP From 2e14308a8d6edf696993068e2d082d4dc7fc35d7 Mon Sep 17 00:00:00 2001 From: Trung Truong Date: Wed, 14 Jan 2015 14:20:24 +1300 Subject: [PATCH 52/52] fix the test script --- mytest/test3-rfclient.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mytest/test3-rfclient.sh b/mytest/test3-rfclient.sh index 635aafd2..86e5ad41 100755 --- a/mytest/test3-rfclient.sh +++ b/mytest/test3-rfclient.sh @@ -195,7 +195,7 @@ reset() { reset 1 trap "reset 0; exit 0" INT -if [ "$ACTION" == "RESET" ]; then +if [ "$ACTION" != "RESET" ]; then echo_bold "-> Starting the management network ($RFBR)..." add_local_br $RFBR ifconfig $RFBR $HOSTVMIP