diff --git a/orchagent/muxorch.cpp b/orchagent/muxorch.cpp index ea3ade347c..ce6bf2baf2 100644 --- a/orchagent/muxorch.cpp +++ b/orchagent/muxorch.cpp @@ -744,6 +744,8 @@ void MuxNbrHandler::update(NextHopKey nh, sai_object_id_t tunnelId, bool add, Mu bool MuxNbrHandler::enable(bool update_rt) { NeighborEntry neigh; + std::list neigh_ctx_list; + std::list route_ctx_list; auto it = neighbors_.begin(); while (it != neighbors_.end()) @@ -751,13 +753,21 @@ bool MuxNbrHandler::enable(bool update_rt) SWSS_LOG_INFO("Enabling neigh %s on %s", it->first.to_string().c_str(), alias_.c_str()); neigh = NeighborEntry(it->first, alias_); - if (!gNeighOrch->enableNeighbor(neigh)) - { - SWSS_LOG_INFO("Enabling neigh failed for %s", neigh.ip_address.to_string().c_str()); - return false; - } + // Create neighbor context with bulk_op enabled + neigh_ctx_list.push_back(NeighborContext(neigh, true)); + it++; + } + + if (!gNeighOrch->enableNeighbors(neigh_ctx_list)) + { + return false; + } + it = neighbors_.begin(); + while (it != neighbors_.end()) + { /* Update NH to point to learned neighbor */ + neigh = NeighborEntry(it->first, alias_); it->second = gNeighOrch->getLocalNextHopId(neigh); /* Reprogram route */ @@ -795,22 +805,26 @@ bool MuxNbrHandler::enable(bool update_rt) IpPrefix pfx = it->first.to_string(); if (update_rt) { - if (remove_route(pfx) != SAI_STATUS_SUCCESS) - { - return false; - } + route_ctx_list.push_back(MuxRouteBulkContext(pfx)); updateTunnelRoute(nh_key, false); } it++; } + if (update_rt && !removeRoutes(route_ctx_list)) + { + return false; + } + return true; } bool MuxNbrHandler::disable(sai_object_id_t tnh) { NeighborEntry neigh; + std::list neigh_ctx_list; + std::list route_ctx_list; auto it = neighbors_.begin(); while (it != neighbors_.end()) @@ -852,21 +866,25 @@ bool MuxNbrHandler::disable(sai_object_id_t tnh) updateTunnelRoute(nh_key, true); IpPrefix pfx = it->first.to_string(); - if (create_route(pfx, it->second) != SAI_STATUS_SUCCESS) - { - return false; - } + route_ctx_list.push_back(MuxRouteBulkContext(pfx, it->second)); neigh = NeighborEntry(it->first, alias_); - if (!gNeighOrch->disableNeighbor(neigh)) - { - SWSS_LOG_INFO("Disabling neigh failed for %s", neigh.ip_address.to_string().c_str()); - return false; - } + // Create neighbor context with bulk_op enabled + neigh_ctx_list.push_back(NeighborContext(neigh, true)); it++; } + if (!addRoutes(route_ctx_list)) + { + return false; + } + + if (!gNeighOrch->disableNeighbors(neigh_ctx_list)) + { + return false; + } + return true; } @@ -881,6 +899,141 @@ sai_object_id_t MuxNbrHandler::getNextHopId(const NextHopKey nhKey) return SAI_NULL_OBJECT_ID; } +bool MuxNbrHandler::addRoutes(std::list& bulk_ctx_list) +{ + sai_status_t status; + bool ret = true; + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + auto& object_statuses = ctx->object_statuses; + sai_route_entry_t route_entry; + route_entry.switch_id = gSwitchId; + route_entry.vr_id = gVirtualRouterId; + copy(route_entry.destination, ctx->pfx); + subnet(route_entry.destination, route_entry.destination); + + SWSS_LOG_INFO("Adding route entry %s, nh %" PRIx64 " to bulker", ctx->pfx.getIp().to_string().c_str(), ctx->nh); + + object_statuses.emplace_back(); + sai_attribute_t attr; + vector attrs; + + attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION; + attr.value.s32 = SAI_PACKET_ACTION_FORWARD; + attrs.push_back(attr); + + attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; + attr.value.oid = ctx->nh; + attrs.push_back(attr); + + status = gRouteBulker.create_entry(&object_statuses.back(), &route_entry, (uint32_t)attrs.size(), attrs.data()); + } + + gRouteBulker.flush(); + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + auto& object_statuses = ctx->object_statuses; + auto it_status = object_statuses.begin(); + status = *it_status++; + + sai_route_entry_t route_entry; + route_entry.switch_id = gSwitchId; + route_entry.vr_id = gVirtualRouterId; + copy(route_entry.destination, ctx->pfx); + subnet(route_entry.destination, route_entry.destination); + + if (status != SAI_STATUS_SUCCESS) + { + if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) { + SWSS_LOG_INFO("Tunnel route to %s already exists", ctx->pfx.to_string().c_str()); + continue; + } + SWSS_LOG_ERROR("Failed to create tunnel route %s,nh %" PRIx64 " rv:%d", + ctx->pfx.getIp().to_string().c_str(), ctx->nh, status); + ret = false; + continue; + } + + if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); + } + else + { + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); + } + + SWSS_LOG_NOTICE("Created tunnel route to %s ", ctx->pfx.to_string().c_str()); + } + + gRouteBulker.clear(); + return ret; +} + +bool MuxNbrHandler::removeRoutes(std::list& bulk_ctx_list) +{ + sai_status_t status; + bool ret = true; + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + auto& object_statuses = ctx->object_statuses; + sai_route_entry_t route_entry; + route_entry.switch_id = gSwitchId; + route_entry.vr_id = gVirtualRouterId; + copy(route_entry.destination, ctx->pfx); + subnet(route_entry.destination, route_entry.destination); + + SWSS_LOG_INFO("Removing route entry %s, nh %" PRIx64 "", ctx->pfx.getIp().to_string().c_str(), ctx->nh); + + object_statuses.emplace_back(); + status = gRouteBulker.remove_entry(&object_statuses.back(), &route_entry); + } + + gRouteBulker.flush(); + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + auto& object_statuses = ctx->object_statuses; + auto it_status = object_statuses.begin(); + status = *it_status++; + + sai_route_entry_t route_entry; + route_entry.switch_id = gSwitchId; + route_entry.vr_id = gVirtualRouterId; + copy(route_entry.destination, ctx->pfx); + subnet(route_entry.destination, route_entry.destination); + + if (status != SAI_STATUS_SUCCESS) + { + if (status == SAI_STATUS_ITEM_NOT_FOUND) { + SWSS_LOG_INFO("Tunnel route to %s already removed", ctx->pfx.to_string().c_str()); + continue; + } + SWSS_LOG_ERROR("Failed to remove tunnel route %s, rv:%d", + ctx->pfx.getIp().to_string().c_str(), status); + ret = false; + continue; + } + + if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); + } + else + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); + } + + SWSS_LOG_NOTICE("Removed tunnel route to %s ", ctx->pfx.to_string().c_str()); + } + + gRouteBulker.clear(); + return ret; +} + void MuxNbrHandler::updateTunnelRoute(NextHopKey nh, bool add) { MuxOrch* mux_orch = gDirectory.get(); diff --git a/orchagent/muxorch.h b/orchagent/muxorch.h index 22f01ce27d..3a6d165db4 100644 --- a/orchagent/muxorch.h +++ b/orchagent/muxorch.h @@ -10,6 +10,7 @@ #include "tunneldecaporch.h" #include "aclorch.h" #include "neighorch.h" +#include "bulker.h" enum MuxState { @@ -35,6 +36,26 @@ enum MuxCableType ACTIVE_ACTIVE }; +struct MuxRouteBulkContext +{ + std::deque object_statuses; // Bulk statuses + IpPrefix pfx; // Route prefix + sai_object_id_t nh; // nexthop id + + MuxRouteBulkContext(IpPrefix pfx) + : pfx(pfx) + { + } + + MuxRouteBulkContext(IpPrefix pfx, sai_object_id_t nh) + : pfx(pfx), nh(nh) + { + } +}; + +extern size_t gMaxBulkSize; +extern sai_route_api_t* sai_route_api; + // Forward Declarations class MuxOrch; class MuxCableOrch; @@ -64,7 +85,7 @@ typedef std::map MuxNeighbor; class MuxNbrHandler { public: - MuxNbrHandler() = default; + MuxNbrHandler() : gRouteBulker(sai_route_api, gMaxBulkSize) {}; bool enable(bool update_rt); bool disable(sai_object_id_t); @@ -75,11 +96,15 @@ class MuxNbrHandler string getAlias() const { return alias_; }; private: + bool removeRoutes(std::list& bulk_ctx_list); + bool addRoutes(std::list& bulk_ctx_list); + inline void updateTunnelRoute(NextHopKey, bool = true); private: MuxNeighbor neighbors_; string alias_; + EntityBulker gRouteBulker; }; // Mux Cable object diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index 006f456a1c..df96405791 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -22,10 +22,12 @@ extern Directory gDirectory; extern string gMySwitchType; extern int32_t gVoqMySwitchId; extern BfdOrch *gBfdOrch; +extern size_t gMaxBulkSize; const int neighorch_pri = 30; NeighOrch::NeighOrch(DBConnector *appDb, string tableName, IntfsOrch *intfsOrch, FdbOrch *fdbOrch, PortsOrch *portsOrch, DBConnector *chassisAppDb) : + gNeighBulker(sai_neighbor_api, gMaxBulkSize), Orch(appDb, tableName, neighorch_pri), m_intfsOrch(intfsOrch), m_fdbOrch(fdbOrch), @@ -793,6 +795,8 @@ void NeighOrch::doTask(Consumer &consumer) NeighborEntry neighbor_entry = { ip_address, alias }; + NeighborContext ctx = NeighborContext(neighbor_entry); + if (op == SET_COMMAND) { Port p; @@ -818,6 +822,8 @@ void NeighOrch::doTask(Consumer &consumer) mac_address = MacAddress(fvValue(*i)); } + ctx.mac = mac_address; + bool nbr_not_found = (m_syncdNeighbors.find(neighbor_entry) == m_syncdNeighbors.end()); if (nbr_not_found || m_syncdNeighbors[neighbor_entry].mac != mac_address) { @@ -846,7 +852,7 @@ void NeighOrch::doTask(Consumer &consumer) it = consumer.m_toSync.erase(it); } } - else if (addNeighbor(neighbor_entry, mac_address)) + else if (addNeighbor(ctx)) { it = consumer.m_toSync.erase(it); } @@ -877,7 +883,7 @@ void NeighOrch::doTask(Consumer &consumer) { if (m_syncdNeighbors.find(neighbor_entry) != m_syncdNeighbors.end()) { - if (removeNeighbor(neighbor_entry)) + if (removeNeighbor(ctx)) { it = consumer.m_toSync.erase(it); } @@ -898,13 +904,18 @@ void NeighOrch::doTask(Consumer &consumer) } } -bool NeighOrch::addNeighbor(const NeighborEntry &neighborEntry, const MacAddress &macAddress) +bool NeighOrch::addNeighbor(NeighborContext& ctx) { SWSS_LOG_ENTER(); sai_status_t status; + auto& object_statuses = ctx.object_statuses; + + const MacAddress &macAddress = ctx.mac; + const NeighborEntry neighborEntry = ctx.neighborEntry; IpAddress ip_address = neighborEntry.ip_address; string alias = neighborEntry.alias; + bool bulk_op = ctx.bulk_op; sai_object_id_t rif_id = m_intfsOrch->getRouterIntfsId(alias); if (rif_id == SAI_NULL_OBJECT_ID) @@ -973,7 +984,8 @@ bool NeighOrch::addNeighbor(const NeighborEntry &neighborEntry, const MacAddress SWSS_LOG_NOTICE("Neighbor %s already learned on %s in VRF %s, removing before adding new neighbor", ip_address.to_string().c_str(), vlan_port.c_str(), vrf_name.c_str()); } - if (!removeNeighbor(temp_entry)) + NeighborContext removeContext = NeighborContext(temp_entry); + if (!removeNeighbor(removeContext)) { SWSS_LOG_ERROR("Failed to remove neighbor %s on %s", ip_address.to_string().c_str(), vlan_port.c_str()); return false; @@ -995,6 +1007,15 @@ bool NeighOrch::addNeighbor(const NeighborEntry &neighborEntry, const MacAddress if (!hw_config && mux_orch->isNeighborActive(ip_address, macAddress, alias)) { + // Using bulker, return and post-process later + if (bulk_op) + { + SWSS_LOG_INFO("Adding neighbor entry %s on %s to bulker.", ip_address.to_string().c_str(), alias.c_str()); + object_statuses.emplace_back(); + gNeighBulker.create_entry(&object_statuses.back(), &neighbor_entry, (uint32_t)neighbor_attrs.size(), neighbor_attrs.data()); + return true; + } + status = sai_neighbor_api->create_neighbor_entry(&neighbor_entry, (uint32_t)neighbor_attrs.size(), neighbor_attrs.data()); if (status != SAI_STATUS_SUCCESS) @@ -1091,13 +1112,17 @@ bool NeighOrch::addNeighbor(const NeighborEntry &neighborEntry, const MacAddress return true; } -bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) +bool NeighOrch::removeNeighbor(NeighborContext& ctx, bool disable) { SWSS_LOG_ENTER(); sai_status_t status; - IpAddress ip_address = neighborEntry.ip_address; + auto& object_statuses = ctx.object_statuses; + + const NeighborEntry neighborEntry = ctx.neighborEntry; string alias = neighborEntry.alias; + IpAddress ip_address = neighborEntry.ip_address; + bool bulk_op = ctx.bulk_op; NextHopKey nexthop = { ip_address, alias }; if(m_intfsOrch->isRemoteSystemPortIntf(alias)) @@ -1168,6 +1193,13 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) SWSS_LOG_NOTICE("Removed next hop %s on %s", ip_address.to_string().c_str(), alias.c_str()); + if (bulk_op) + { + object_statuses.emplace_back(); + gNeighBulker.remove_entry(&object_statuses.back(), &neighbor_entry); + return true; + } + status = sai_neighbor_api->remove_neighbor_entry(&neighbor_entry); if (status != SAI_STATUS_SUCCESS) { @@ -1227,6 +1259,185 @@ bool NeighOrch::removeNeighbor(const NeighborEntry &neighborEntry, bool disable) return true; } +/* Process bulk ctx entry and enable the neigbor */ +bool NeighOrch::processBulkEnableNeighbor(NeighborContext& ctx) +{ + SWSS_LOG_ENTER(); + + const auto& object_statuses = ctx.object_statuses; + auto it_status = object_statuses.begin(); + sai_status_t status; + + const MacAddress &macAddress = ctx.mac; + const NeighborEntry neighborEntry = ctx.neighborEntry; + string alias = neighborEntry.alias; + IpAddress ip_address = neighborEntry.ip_address; + + if (!ctx.bulk_op) + { + SWSS_LOG_INFO("Not a bulk entry for %s on %s", ip_address.to_string().c_str(), alias.c_str()); + return true; + } + + SWSS_LOG_INFO("Checking neighbor create entry status %s on %s.", ip_address.to_string().c_str(), alias.c_str()); + + sai_object_id_t rif_id = m_intfsOrch->getRouterIntfsId(alias); + if (rif_id == SAI_NULL_OBJECT_ID) + { + SWSS_LOG_INFO("Failed to get rif_id for %s", alias.c_str()); + return false; + } + + sai_neighbor_entry_t neighbor_entry; + neighbor_entry.rif_id = rif_id; + neighbor_entry.switch_id = gSwitchId; + copy(neighbor_entry.ip_address, ip_address); + + MuxOrch* mux_orch = gDirectory.get(); + if (mux_orch->isNeighborActive(ip_address, macAddress, alias)) + { + status = *it_status++; + if (status != SAI_STATUS_SUCCESS) + { + if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) + { + SWSS_LOG_INFO("Neighbor exists: neighbor %s on %s, skipping: status:%s", + macAddress.to_string().c_str(), alias.c_str(), sai_serialize_status(status).c_str()); + return true; + } + else + { + SWSS_LOG_ERROR("Failed to create neighbor %s on %s, status:%s", + macAddress.to_string().c_str(), alias.c_str(), sai_serialize_status(status).c_str()); + task_process_status handle_status = handleSaiCreateStatus(SAI_API_NEIGHBOR, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + } + + SWSS_LOG_NOTICE("Created neighbor ip %s, %s on %s", ip_address.to_string().c_str(), + macAddress.to_string().c_str(), alias.c_str()); + + m_intfsOrch->increaseRouterIntfsRefCount(alias); + + if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR); + } + else + { + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR); + } + + if (!addNextHop(NextHopKey(ip_address, alias))) + { + status = sai_neighbor_api->remove_neighbor_entry(&neighbor_entry); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove neighbor %s on %s, rv:%d", + macAddress.to_string().c_str(), alias.c_str(), status); + task_process_status handle_status = handleSaiRemoveStatus(SAI_API_NEIGHBOR, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + m_intfsOrch->decreaseRouterIntfsRefCount(alias); + + if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR); + } + else + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR); + } + + return false; + } + } + + m_syncdNeighbors[neighborEntry] = { macAddress, true }; + + NeighborUpdate update = { neighborEntry, macAddress, true }; + notify(SUBJECT_TYPE_NEIGH_CHANGE, static_cast(&update)); + + return true; +} + +/* Process bulk ctx entry and disable the neigbor */ +bool NeighOrch::processBulkDisableNeighbor(NeighborContext& ctx) +{ + SWSS_LOG_ENTER(); + + const auto& object_statuses = ctx.object_statuses; + auto it_status = object_statuses.begin(); + sai_status_t status; + + const NeighborEntry neighborEntry = ctx.neighborEntry; + string alias = neighborEntry.alias; + IpAddress ip_address = neighborEntry.ip_address; + + if (m_syncdNeighbors.find(neighborEntry) == m_syncdNeighbors.end()) + { + return true; + } + + SWSS_LOG_INFO("Checking neighbor remove entry status %s on %s.", ip_address.to_string().c_str(), m_syncdNeighbors[neighborEntry].mac.to_string().c_str()); + + if (isHwConfigured(neighborEntry)) + { + sai_object_id_t rif_id = m_intfsOrch->getRouterIntfsId(alias); + + sai_neighbor_entry_t neighbor_entry; + neighbor_entry.rif_id = rif_id; + neighbor_entry.switch_id = gSwitchId; + copy(neighbor_entry.ip_address, ip_address); + + status = *it_status++; + if (status != SAI_STATUS_SUCCESS) + { + if (status == SAI_STATUS_ITEM_NOT_FOUND) + { + SWSS_LOG_NOTICE("Bulk remove entry skipped, neighbor %s on %s already removed, rv:%d", + m_syncdNeighbors[neighborEntry].mac.to_string().c_str(), alias.c_str(), status); + } + else + { + SWSS_LOG_ERROR("Failed to remove neighbor %s on %s, rv:%d", + m_syncdNeighbors[neighborEntry].mac.to_string().c_str(), alias.c_str(), status); + task_process_status handle_status = handleSaiRemoveStatus(SAI_API_NEIGHBOR, status); + if (handle_status != task_success) + { + return parseHandleSaiStatusFailure(handle_status); + } + } + } + else + { + if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4) + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR); + } + else + { + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR); + } + + removeNextHop(ip_address, alias); + m_intfsOrch->decreaseRouterIntfsRefCount(alias); + SWSS_LOG_NOTICE("Removed neighbor %s on %s", + m_syncdNeighbors[neighborEntry].mac.to_string().c_str(), alias.c_str()); + } + } + + /* Do not delete entry from cache for disable request */ + m_syncdNeighbors[neighborEntry].hw_configured = false; + return true; +} + bool NeighOrch::isHwConfigured(const NeighborEntry& neighborEntry) { if (m_syncdNeighbors.find(neighborEntry) == m_syncdNeighbors.end()) @@ -1253,7 +1464,11 @@ bool NeighOrch::enableNeighbor(const NeighborEntry& neighborEntry) return true; } - return addNeighbor(neighborEntry, m_syncdNeighbors[neighborEntry].mac); + NeighborEntry neigh = neighborEntry; + NeighborContext ctx = NeighborContext(neigh); + ctx.mac = m_syncdNeighbors[neighborEntry].mac; + + return addNeighbor(ctx); } bool NeighOrch::disableNeighbor(const NeighborEntry& neighborEntry) @@ -1272,7 +1487,108 @@ bool NeighOrch::disableNeighbor(const NeighborEntry& neighborEntry) return true; } - return removeNeighbor(neighborEntry, true); + NeighborContext ctx = NeighborContext(neighborEntry); + + return removeNeighbor(ctx, true); +} + +/* enable neighbors using bulker */ +bool NeighOrch::enableNeighbors(std::list& bulk_ctx_list) +{ + bool ret = true; + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + const NeighborEntry& neighborEntry = ctx->neighborEntry; + ctx->mac = m_syncdNeighbors[neighborEntry].mac; + + if (m_syncdNeighbors.find(neighborEntry) == m_syncdNeighbors.end()) + { + SWSS_LOG_INFO("Neighbor %s not found", neighborEntry.ip_address.to_string().c_str()); + continue; + } + + if (isHwConfigured(neighborEntry)) + { + SWSS_LOG_INFO("Neighbor %s is already programmed to HW", neighborEntry.ip_address.to_string().c_str()); + continue; + } + + SWSS_LOG_NOTICE("Neighbor enable request for %s ", neighborEntry.ip_address.to_string().c_str()); + + if(!addNeighbor(*ctx)) + { + SWSS_LOG_ERROR("Neighbor %s create entry failed.", neighborEntry.ip_address.to_string().c_str()); + continue; + } + } + + gNeighBulker.flush(); + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + if (ctx->object_statuses.empty()) + { + continue; + } + + const NeighborEntry& neighborEntry = ctx->neighborEntry; + if (!processBulkEnableNeighbor(*ctx)) + { + SWSS_LOG_INFO("Enable neighbor failed for %s", neighborEntry.ip_address.to_string().c_str()); + /* finish processing bulk entries */ + ret = false; + } + } + + gNeighBulker.clear(); + return ret; +} + +/* disable neighbors using bulker */ +bool NeighOrch::disableNeighbors(std::list& bulk_ctx_list) +{ + bool ret = true; + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + const NeighborEntry& neighborEntry = ctx->neighborEntry; + ctx->mac = m_syncdNeighbors[neighborEntry].mac; + + if (m_syncdNeighbors.find(neighborEntry) == m_syncdNeighbors.end()) + { + SWSS_LOG_INFO("Neighbor %s not found", neighborEntry.ip_address.to_string().c_str()); + continue; + } + + SWSS_LOG_NOTICE("Neighbor disable request for %s ", neighborEntry.ip_address.to_string().c_str()); + + if(!removeNeighbor(*ctx, true)) + { + SWSS_LOG_ERROR("Neighbor %s remove entry failed.", neighborEntry.ip_address.to_string().c_str()); + } + } + + gNeighBulker.flush(); + + for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++) + { + if (ctx->object_statuses.empty()) + { + continue; + } + + const NeighborEntry& neighborEntry = ctx->neighborEntry; + if (!processBulkDisableNeighbor(*ctx)) + { + SWSS_LOG_INFO("Disable neighbor failed for %s", neighborEntry.ip_address.to_string().c_str()); + /* finish processing bulk entries but return false */ + ret = false; + } + } + + gNeighBulker.clear(); + return ret; } sai_object_id_t NeighOrch::addTunnelNextHop(const NextHopKey& nh) @@ -1454,7 +1770,8 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer) SWSS_LOG_NOTICE("VOQ encap index set failed for neighbor %s. Removing and re-adding", kfvKey(t).c_str()); //Remove neigh from SAI - if (removeNeighbor(neighbor_entry)) + NeighborContext ctx = NeighborContext(neighbor_entry); + if (removeNeighbor(ctx)) { //neigh successfully deleted from SAI. Set STATE DB to signal to remove entries from kernel m_stateSystemNeighTable->del(state_key); @@ -1485,7 +1802,9 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer) } //Add neigh to SAI - if (addNeighbor(neighbor_entry, mac_address)) + NeighborContext ctx = NeighborContext(neighbor_entry); + ctx.mac = mac_address; + if (addNeighbor(ctx)) { //neigh successfully added to SAI. Set STATE DB to signal kernel programming by neighbor manager @@ -1538,7 +1857,8 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer) if (m_syncdNeighbors.find(neighbor_entry) != m_syncdNeighbors.end()) { //Remove neigh from SAI - if (removeNeighbor(neighbor_entry)) + NeighborContext ctx = NeighborContext(neighbor_entry); + if (removeNeighbor(ctx)) { //neigh successfully deleted from SAI. Set STATE DB to signal to remove entries from kernel m_stateSystemNeighTable->del(state_key); diff --git a/orchagent/neighorch.h b/orchagent/neighorch.h index f44741fa37..0b59181db1 100644 --- a/orchagent/neighorch.h +++ b/orchagent/neighorch.h @@ -12,6 +12,7 @@ #include "producerstatetable.h" #include "schema.h" #include "bfdorch.h" +#include "bulker.h" #define NHFLAGS_IFDOWN 0x1 // nexthop's outbound i/f is down @@ -43,6 +44,27 @@ struct NeighborUpdate bool add; }; +/* + * Keeps track of neighbor entry information primarily for bulk operations + */ +struct NeighborContext +{ + NeighborEntry neighborEntry; // neighbor entry to process + std::deque object_statuses; // bulk statuses + MacAddress mac; // neighbor mac + bool bulk_op = false; // use bulker (only for mux use for now) + + NeighborContext(NeighborEntry neighborEntry) + : neighborEntry(neighborEntry) + { + } + + NeighborContext(NeighborEntry neighborEntry, bool bulk_op) + : neighborEntry(neighborEntry), bulk_op(bulk_op) + { + } +}; + class NeighOrch : public Orch, public Subject, public Observer { public: @@ -66,6 +88,8 @@ class NeighOrch : public Orch, public Subject, public Observer bool enableNeighbor(const NeighborEntry&); bool disableNeighbor(const NeighborEntry&); + bool enableNeighbors(std::list&); + bool disableNeighbors(std::list&); bool isHwConfigured(const NeighborEntry&); sai_object_id_t addTunnelNextHop(const NextHopKey&); @@ -95,10 +119,14 @@ class NeighOrch : public Orch, public Subject, public Observer std::set m_neighborToResolve; + EntityBulker gNeighBulker; + bool removeNextHop(const IpAddress&, const string&); - bool addNeighbor(const NeighborEntry&, const MacAddress&); - bool removeNeighbor(const NeighborEntry&, bool disable = false); + bool addNeighbor(NeighborContext& ctx); + bool removeNeighbor(NeighborContext& ctx, bool disable = false); + bool processBulkEnableNeighbor(NeighborContext& ctx); + bool processBulkDisableNeighbor(NeighborContext& ctx); bool setNextHopFlag(const NextHopKey &, const uint32_t); bool clearNextHopFlag(const NextHopKey &, const uint32_t); diff --git a/orchagent/p4orch/tests/mock_sai_neighbor.h b/orchagent/p4orch/tests/mock_sai_neighbor.h index cd8f2aa0a9..4355831d36 100644 --- a/orchagent/p4orch/tests/mock_sai_neighbor.h +++ b/orchagent/p4orch/tests/mock_sai_neighbor.h @@ -16,6 +16,12 @@ class MockSaiNeighbor MOCK_METHOD1(remove_neighbor_entry, sai_status_t(_In_ const sai_neighbor_entry_t *neighbor_entry)); + MOCK_METHOD6(create_neighbor_entries, sai_status_t(_In_ uint32_t object_count, _In_ const sai_neighbor_entry_t *neighbor_entry, _In_ const uint32_t *attr_count, + _In_ const sai_attribute_t **attr_list, _In_ sai_bulk_op_error_mode_t mode, _Out_ sai_status_t *object_statuses)); + + MOCK_METHOD4(remove_neighbor_entries, sai_status_t(_In_ uint32_t object_count, _In_ const sai_neighbor_entry_t *neighbor_entry, _In_ sai_bulk_op_error_mode_t mode, + _Out_ sai_status_t *object_statuses)); + MOCK_METHOD2(set_neighbor_entry_attribute, sai_status_t(_In_ const sai_neighbor_entry_t *neighbor_entry, _In_ const sai_attribute_t *attr)); @@ -37,6 +43,18 @@ sai_status_t mock_remove_neighbor_entry(_In_ const sai_neighbor_entry_t *neighbo return mock_sai_neighbor->remove_neighbor_entry(neighbor_entry); } +sai_status_t mock_create_neighbor_entries(_In_ uint32_t object_count, _In_ const sai_neighbor_entry_t *neighbor_entry, _In_ const uint32_t *attr_count, + _In_ const sai_attribute_t **attr_list, _In_ sai_bulk_op_error_mode_t mode, _Out_ sai_status_t *object_statuses) +{ + return mock_sai_neighbor->create_neighbor_entries(object_count, neighbor_entry, attr_count, attr_list, mode, object_statuses); +} + +sai_status_t mock_remove_neighbor_entries(_In_ uint32_t object_count, _In_ const sai_neighbor_entry_t *neighbor_entry, _In_ sai_bulk_op_error_mode_t mode, + _Out_ sai_status_t *object_statuses) +{ + return mock_sai_neighbor->remove_neighbor_entries(object_count, neighbor_entry, mode, object_statuses); +} + sai_status_t mock_set_neighbor_entry_attribute(_In_ const sai_neighbor_entry_t *neighbor_entry, _In_ const sai_attribute_t *attr) { diff --git a/orchagent/p4orch/tests/neighbor_manager_test.cpp b/orchagent/p4orch/tests/neighbor_manager_test.cpp index 4db1db873e..7523701cb7 100644 --- a/orchagent/p4orch/tests/neighbor_manager_test.cpp +++ b/orchagent/p4orch/tests/neighbor_manager_test.cpp @@ -124,6 +124,8 @@ class NeighborManagerTest : public ::testing::Test mock_sai_neighbor = &mock_sai_neighbor_; sai_neighbor_api->create_neighbor_entry = mock_create_neighbor_entry; sai_neighbor_api->remove_neighbor_entry = mock_remove_neighbor_entry; + sai_neighbor_api->create_neighbor_entries = mock_create_neighbor_entries; + sai_neighbor_api->remove_neighbor_entries = mock_remove_neighbor_entries; sai_neighbor_api->set_neighbor_entry_attribute = mock_set_neighbor_entry_attribute; sai_neighbor_api->get_neighbor_entry_attribute = mock_get_neighbor_entry_attribute; } diff --git a/tests/mock_tests/aclorch_ut.cpp b/tests/mock_tests/aclorch_ut.cpp index 8005199935..4a92d65c80 100644 --- a/tests/mock_tests/aclorch_ut.cpp +++ b/tests/mock_tests/aclorch_ut.cpp @@ -24,6 +24,7 @@ extern sai_port_api_t *sai_port_api; extern sai_vlan_api_t *sai_vlan_api; extern sai_bridge_api_t *sai_bridge_api; extern sai_route_api_t *sai_route_api; +extern sai_route_api_t *sai_neighbor_api; extern sai_mpls_api_t *sai_mpls_api; extern sai_next_hop_group_api_t* sai_next_hop_group_api; extern string gMySwitchType; @@ -318,6 +319,7 @@ namespace aclorch_test sai_api_query(SAI_API_PORT, (void **)&sai_port_api); sai_api_query(SAI_API_VLAN, (void **)&sai_vlan_api); sai_api_query(SAI_API_ROUTE, (void **)&sai_route_api); + sai_api_query(SAI_API_NEIGHBOR, (void **)&sai_neighbor_api); sai_api_query(SAI_API_MPLS, (void **)&sai_mpls_api); sai_api_query(SAI_API_ACL, (void **)&sai_acl_api); sai_api_query(SAI_API_NEXT_HOP_GROUP, (void **)&sai_next_hop_group_api); @@ -490,6 +492,7 @@ namespace aclorch_test sai_vlan_api = nullptr; sai_bridge_api = nullptr; sai_route_api = nullptr; + sai_neighbor_api = nullptr; sai_mpls_api = nullptr; } diff --git a/tests/mock_tests/bulker_ut.cpp b/tests/mock_tests/bulker_ut.cpp index 6210cc0969..dc5ad78776 100644 --- a/tests/mock_tests/bulker_ut.cpp +++ b/tests/mock_tests/bulker_ut.cpp @@ -2,6 +2,7 @@ #include "bulker.h" extern sai_route_api_t *sai_route_api; +extern sai_neighbor_api_t *sai_neighbor_api; namespace bulker_test { @@ -17,12 +18,18 @@ namespace bulker_test { ASSERT_EQ(sai_route_api, nullptr); sai_route_api = new sai_route_api_t(); + + ASSERT_EQ(sai_neighbor_api, nullptr); + sai_neighbor_api = new sai_neighbor_api_t(); } void TearDown() override { delete sai_route_api; sai_route_api = nullptr; + + delete sai_neighbor_api; + sai_neighbor_api = nullptr; } }; @@ -142,4 +149,28 @@ namespace bulker_test // Confirm route entry is not pending removal ASSERT_FALSE(gRouteBulker.bulk_entry_pending_removal(route_entry_non_remove)); } + + TEST_F(BulkerTest, NeighborBulker) + { + // Create bulker + EntityBulker gNeighBulker(sai_neighbor_api, 1000); + deque object_statuses; + + // Check max bulk size + ASSERT_EQ(gNeighBulker.max_bulk_size, 1000); + + // Create a dummy neighbor entry + sai_neighbor_entry_t neighbor_entry_remove; + neighbor_entry_remove.ip_address.addr_family = SAI_IP_ADDR_FAMILY_IPV4; + neighbor_entry_remove.ip_address.addr.ip4 = 0x10000001; + neighbor_entry_remove.rif_id = 0x0; + neighbor_entry_remove.switch_id = 0x0; + + // Put neighbor entry into remove + object_statuses.emplace_back(); + gNeighBulker.remove_entry(&object_statuses.back(), &neighbor_entry_remove); + + // Confirm neighbor entry is pending removal + ASSERT_TRUE(gNeighBulker.bulk_entry_pending_removal(neighbor_entry_remove)); + } } diff --git a/tests/mock_tests/mock_sai_api.h b/tests/mock_tests/mock_sai_api.h index 7819b5b126..58e3c9da23 100644 --- a/tests/mock_tests/mock_sai_api.h +++ b/tests/mock_tests/mock_sai_api.h @@ -24,8 +24,12 @@ EXTERN_MOCK_FNS #define CREATE_PARAMS(sai_object_type) _In_ const sai_##sai_object_type##_entry_t *sai_object_type##_entry, _In_ uint32_t attr_count, _In_ const sai_attribute_t *attr_list #define REMOVE_PARAMS(sai_object_type) _In_ const sai_##sai_object_type##_entry_t *sai_object_type##_entry +#define CREATE_BULK_PARAMS(sai_object_type) _In_ uint32_t object_count, _In_ const sai_##sai_object_type##_entry_t *sai_object_type##_entry, _In_ const uint32_t *attr_count, _In_ const sai_attribute_t **attr_list, _In_ sai_bulk_op_error_mode_t mode, _Out_ sai_status_t *object_statuses +#define REMOVE_BULK_PARAMS(sai_object_type) _In_ uint32_t object_count, _In_ const sai_##sai_object_type##_entry_t *sai_object_type##_entry, _In_ sai_bulk_op_error_mode_t mode, _In_ sai_status_t *object_statuses #define CREATE_ARGS(sai_object_type) sai_object_type##_entry, attr_count, attr_list #define REMOVE_ARGS(sai_object_type) sai_object_type##_entry +#define CREATE_BULK_ARGS(sai_object_type) object_count, sai_object_type##_entry, attr_count, attr_list, mode, object_statuses +#define REMOVE_BULK_ARGS(sai_object_type) object_count, sai_object_type##_entry, mode, object_statuses #define GENERIC_CREATE_PARAMS(sai_object_type) _Out_ sai_object_id_t *sai_object_type##_id, _In_ sai_object_id_t switch_id, _In_ uint32_t attr_count, _In_ const sai_attribute_t *attr_list #define GENERIC_REMOVE_PARAMS(sai_object_type) _In_ sai_object_id_t sai_object_type##_id #define GENERIC_CREATE_ARGS(sai_object_type) sai_object_type##_id, switch_id, attr_count, attr_list @@ -42,8 +46,8 @@ The macro DEFINE_SAI_API_MOCK will perform the steps to mock the SAI API for the 7. Define a method to remove the mock */ #define DEFINE_SAI_API_MOCK(sai_object_type) \ - static sai_##sai_object_type##_api_t *old_sai_##sai_object_type##_api; \ - static sai_##sai_object_type##_api_t ut_sai_##sai_object_type##_api; \ + static sai_##sai_object_type##_api_t *old_sai_##sai_object_type##_api; \ + static sai_##sai_object_type##_api_t ut_sai_##sai_object_type##_api; \ class mock_sai_##sai_object_type##_api_t \ { \ public: \ @@ -59,20 +63,40 @@ The macro DEFINE_SAI_API_MOCK will perform the steps to mock the SAI API for the [this](REMOVE_PARAMS(sai_object_type)) { \ return old_sai_##sai_object_type##_api->remove_##sai_object_type##_entry(REMOVE_ARGS(sai_object_type)); \ }); \ + ON_CALL(*this, create_##sai_object_type##_entries) \ + .WillByDefault( \ + [this](CREATE_BULK_PARAMS(sai_object_type)) { \ + return old_sai_##sai_object_type##_api->create_##sai_object_type##_entries(CREATE_BULK_ARGS(sai_object_type)); \ + }); \ + ON_CALL(*this, remove_##sai_object_type##_entries) \ + .WillByDefault( \ + [this](REMOVE_BULK_PARAMS(sai_object_type)) { \ + return old_sai_##sai_object_type##_api->remove_##sai_object_type##_entries(REMOVE_BULK_ARGS(sai_object_type)); \ + }); \ } \ MOCK_METHOD3(create_##sai_object_type##_entry, sai_status_t(CREATE_PARAMS(sai_object_type))); \ MOCK_METHOD1(remove_##sai_object_type##_entry, sai_status_t(REMOVE_PARAMS(sai_object_type))); \ + MOCK_METHOD6(create_##sai_object_type##_entries, sai_status_t(CREATE_BULK_PARAMS(sai_object_type))); \ + MOCK_METHOD4(remove_##sai_object_type##_entries, sai_status_t(REMOVE_BULK_PARAMS(sai_object_type))); \ }; \ - static mock_sai_##sai_object_type##_api_t *mock_sai_##sai_object_type##_api; \ - inline sai_status_t mock_create_##sai_object_type##_entry(CREATE_PARAMS(sai_object_type)) \ + static mock_sai_##sai_object_type##_api_t *mock_sai_##sai_object_type##_api; \ + inline sai_status_t mock_create_##sai_object_type##_entry(CREATE_PARAMS(sai_object_type)) \ { \ return mock_sai_##sai_object_type##_api->create_##sai_object_type##_entry(CREATE_ARGS(sai_object_type)); \ } \ - inline sai_status_t mock_remove_##sai_object_type##_entry(REMOVE_PARAMS(sai_object_type)) \ + inline sai_status_t mock_remove_##sai_object_type##_entry(REMOVE_PARAMS(sai_object_type)) \ { \ return mock_sai_##sai_object_type##_api->remove_##sai_object_type##_entry(REMOVE_ARGS(sai_object_type)); \ } \ - inline void apply_sai_##sai_object_type##_api_mock() \ + inline sai_status_t mock_create_##sai_object_type##_entries(CREATE_BULK_PARAMS(sai_object_type)) \ + { \ + return mock_sai_##sai_object_type##_api->create_##sai_object_type##_entries(CREATE_BULK_ARGS(sai_object_type)); \ + } \ + inline sai_status_t mock_remove_##sai_object_type##_entries(REMOVE_BULK_PARAMS(sai_object_type)) \ + { \ + return mock_sai_##sai_object_type##_api->remove_##sai_object_type##_entries(REMOVE_BULK_ARGS(sai_object_type)); \ + } \ + inline void apply_sai_##sai_object_type##_api_mock() \ { \ mock_sai_##sai_object_type##_api = new NiceMock(); \ \ @@ -82,16 +106,18 @@ The macro DEFINE_SAI_API_MOCK will perform the steps to mock the SAI API for the \ sai_##sai_object_type##_api->create_##sai_object_type##_entry = mock_create_##sai_object_type##_entry; \ sai_##sai_object_type##_api->remove_##sai_object_type##_entry = mock_remove_##sai_object_type##_entry; \ + sai_##sai_object_type##_api->create_##sai_object_type##_entries = mock_create_##sai_object_type##_entries; \ + sai_##sai_object_type##_api->remove_##sai_object_type##_entries = mock_remove_##sai_object_type##_entries; \ } \ - inline void remove_sai_##sai_object_type##_api_mock() \ + inline void remove_sai_##sai_object_type##_api_mock() \ { \ sai_##sai_object_type##_api = old_sai_##sai_object_type##_api; \ delete mock_sai_##sai_object_type##_api; \ } #define DEFINE_SAI_GENERIC_API_MOCK(sai_api_name, sai_object_type) \ - static sai_##sai_api_name##_api_t *old_sai_##sai_api_name##_api; \ - static sai_##sai_api_name##_api_t ut_sai_##sai_api_name##_api; \ + static sai_##sai_api_name##_api_t *old_sai_##sai_api_name##_api; \ + static sai_##sai_api_name##_api_t ut_sai_##sai_api_name##_api; \ class mock_sai_##sai_api_name##_api_t \ { \ public: \ @@ -111,16 +137,16 @@ The macro DEFINE_SAI_API_MOCK will perform the steps to mock the SAI API for the MOCK_METHOD4(create_##sai_object_type, sai_status_t(GENERIC_CREATE_PARAMS(sai_object_type))); \ MOCK_METHOD1(remove_##sai_object_type, sai_status_t(GENERIC_REMOVE_PARAMS(sai_object_type))); \ }; \ - static mock_sai_##sai_api_name##_api_t *mock_sai_##sai_api_name##_api; \ - inline sai_status_t mock_create_##sai_object_type(GENERIC_CREATE_PARAMS(sai_object_type)) \ + static mock_sai_##sai_api_name##_api_t *mock_sai_##sai_api_name##_api; \ + inline sai_status_t mock_create_##sai_object_type(GENERIC_CREATE_PARAMS(sai_object_type)) \ { \ return mock_sai_##sai_api_name##_api->create_##sai_object_type(GENERIC_CREATE_ARGS(sai_object_type)); \ } \ - inline sai_status_t mock_remove_##sai_object_type(GENERIC_REMOVE_PARAMS(sai_object_type)) \ + inline sai_status_t mock_remove_##sai_object_type(GENERIC_REMOVE_PARAMS(sai_object_type)) \ { \ return mock_sai_##sai_api_name##_api->remove_##sai_object_type(GENERIC_REMOVE_ARGS(sai_object_type)); \ } \ - inline void apply_sai_##sai_api_name##_api_mock() \ + inline void apply_sai_##sai_api_name##_api_mock() \ { \ mock_sai_##sai_api_name##_api = new NiceMock(); \ \ @@ -131,7 +157,7 @@ The macro DEFINE_SAI_API_MOCK will perform the steps to mock the SAI API for the sai_##sai_api_name##_api->create_##sai_object_type = mock_create_##sai_object_type; \ sai_##sai_api_name##_api->remove_##sai_object_type = mock_remove_##sai_object_type; \ } \ - inline void remove_sai_##sai_api_name##_api_mock() \ + inline void remove_sai_##sai_api_name##_api_mock() \ { \ sai_##sai_api_name##_api = old_sai_##sai_api_name##_api; \ delete mock_sai_##sai_api_name##_api; \ diff --git a/tests/mock_tests/mux_rollback_ut.cpp b/tests/mock_tests/mux_rollback_ut.cpp index 008b0bd9b5..52aa29d24e 100644 --- a/tests/mock_tests/mux_rollback_ut.cpp +++ b/tests/mock_tests/mux_rollback_ut.cpp @@ -5,6 +5,10 @@ #include "orch.h" #undef protected #include "ut_helper.h" +#define private public +#include "neighorch.h" +#include "muxorch.h" +#undef private #include "mock_orchagent_main.h" #include "mock_sai_api.h" #include "mock_orch_test.h" @@ -19,13 +23,21 @@ namespace mux_rollback_test DEFINE_SAI_API_MOCK(route); DEFINE_SAI_GENERIC_API_MOCK(acl, acl_entry); DEFINE_SAI_GENERIC_API_MOCK(next_hop, next_hop); + using ::testing::_; using namespace std; using namespace mock_orch_test; using ::testing::Return; using ::testing::Throw; + using ::testing::DoAll; + using ::testing::SetArrayArgument; static const string TEST_INTERFACE = "Ethernet4"; + sai_bulk_create_neighbor_entry_fn old_create_neighbor_entries; + sai_bulk_remove_neighbor_entry_fn old_remove_neighbor_entries; + sai_bulk_create_route_entry_fn old_create_route_entries; + sai_bulk_remove_route_entry_fn old_remove_route_entries; + class MuxRollbackTest : public MockOrchTest { protected: @@ -131,41 +143,57 @@ namespace mux_rollback_test INIT_SAI_API_MOCK(acl); INIT_SAI_API_MOCK(next_hop); MockSaiApis(); + old_create_neighbor_entries = gNeighOrch->gNeighBulker.create_entries; + old_remove_neighbor_entries = gNeighOrch->gNeighBulker.remove_entries; + old_create_route_entries = m_MuxCable->nbr_handler_->gRouteBulker.create_entries; + old_remove_route_entries = m_MuxCable->nbr_handler_->gRouteBulker.remove_entries; + gNeighOrch->gNeighBulker.create_entries = mock_create_neighbor_entries; + gNeighOrch->gNeighBulker.remove_entries = mock_remove_neighbor_entries; + m_MuxCable->nbr_handler_->gRouteBulker.create_entries = mock_create_route_entries; + m_MuxCable->nbr_handler_->gRouteBulker.remove_entries = mock_remove_route_entries; } void PreTearDown() override { RestoreSaiApis(); + gNeighOrch->gNeighBulker.create_entries = old_create_neighbor_entries; + gNeighOrch->gNeighBulker.remove_entries = old_remove_neighbor_entries; + m_MuxCable->nbr_handler_->gRouteBulker.create_entries = old_create_route_entries; + m_MuxCable->nbr_handler_->gRouteBulker.remove_entries = old_remove_route_entries; } }; TEST_F(MuxRollbackTest, StandbyToActiveNeighborAlreadyExists) { - EXPECT_CALL(*mock_sai_neighbor_api, create_neighbor_entry) - .WillOnce(Return(SAI_STATUS_ITEM_ALREADY_EXISTS)); + std::vector exp_status{SAI_STATUS_ITEM_ALREADY_EXISTS}; + EXPECT_CALL(*mock_sai_neighbor_api, create_neighbor_entries) + .WillOnce(DoAll(SetArrayArgument<5>(exp_status.begin(), exp_status.end()), Return(SAI_STATUS_ITEM_ALREADY_EXISTS))); SetAndAssertMuxState(ACTIVE_STATE); } TEST_F(MuxRollbackTest, ActiveToStandbyNeighborNotFound) { SetAndAssertMuxState(ACTIVE_STATE); - EXPECT_CALL(*mock_sai_neighbor_api, remove_neighbor_entry) - .WillOnce(Return(SAI_STATUS_ITEM_NOT_FOUND)); + std::vector exp_status{SAI_STATUS_ITEM_NOT_FOUND}; + EXPECT_CALL(*mock_sai_neighbor_api, remove_neighbor_entries) + .WillOnce(DoAll(SetArrayArgument<3>(exp_status.begin(), exp_status.end()), Return(SAI_STATUS_ITEM_NOT_FOUND))); SetAndAssertMuxState(STANDBY_STATE); } TEST_F(MuxRollbackTest, StandbyToActiveRouteNotFound) { - EXPECT_CALL(*mock_sai_route_api, remove_route_entry) - .WillOnce(Return(SAI_STATUS_ITEM_NOT_FOUND)); + std::vector exp_status{SAI_STATUS_ITEM_NOT_FOUND}; + EXPECT_CALL(*mock_sai_route_api, remove_route_entries) + .WillOnce(DoAll(SetArrayArgument<3>(exp_status.begin(), exp_status.end()), Return(SAI_STATUS_ITEM_NOT_FOUND))); SetAndAssertMuxState(ACTIVE_STATE); } TEST_F(MuxRollbackTest, ActiveToStandbyRouteAlreadyExists) { SetAndAssertMuxState(ACTIVE_STATE); - EXPECT_CALL(*mock_sai_route_api, create_route_entry) - .WillOnce(Return(SAI_STATUS_ITEM_ALREADY_EXISTS)); + std::vector exp_status{SAI_STATUS_ITEM_ALREADY_EXISTS}; + EXPECT_CALL(*mock_sai_route_api, create_route_entries) + .WillOnce(DoAll(SetArrayArgument<5>(exp_status.begin(), exp_status.end()), Return(SAI_STATUS_ITEM_ALREADY_EXISTS))); SetAndAssertMuxState(STANDBY_STATE); } @@ -201,7 +229,7 @@ namespace mux_rollback_test TEST_F(MuxRollbackTest, StandbyToActiveRuntimeErrorRollbackToStandby) { - EXPECT_CALL(*mock_sai_route_api, remove_route_entry) + EXPECT_CALL(*mock_sai_route_api, remove_route_entries) .WillOnce(Throw(runtime_error("Mock runtime error"))); SetMuxStateFromAppDb(ACTIVE_STATE); EXPECT_EQ(STANDBY_STATE, m_MuxCable->getState()); @@ -210,7 +238,7 @@ namespace mux_rollback_test TEST_F(MuxRollbackTest, ActiveToStandbyRuntimeErrorRollbackToActive) { SetAndAssertMuxState(ACTIVE_STATE); - EXPECT_CALL(*mock_sai_route_api, create_route_entry) + EXPECT_CALL(*mock_sai_route_api, create_route_entries) .WillOnce(Throw(runtime_error("Mock runtime error"))); SetMuxStateFromAppDb(STANDBY_STATE); EXPECT_EQ(ACTIVE_STATE, m_MuxCable->getState()); @@ -218,7 +246,7 @@ namespace mux_rollback_test TEST_F(MuxRollbackTest, StandbyToActiveLogicErrorRollbackToStandby) { - EXPECT_CALL(*mock_sai_neighbor_api, create_neighbor_entry) + EXPECT_CALL(*mock_sai_neighbor_api, create_neighbor_entries) .WillOnce(Throw(logic_error("Mock logic error"))); SetMuxStateFromAppDb(ACTIVE_STATE); EXPECT_EQ(STANDBY_STATE, m_MuxCable->getState()); @@ -227,7 +255,7 @@ namespace mux_rollback_test TEST_F(MuxRollbackTest, ActiveToStandbyLogicErrorRollbackToActive) { SetAndAssertMuxState(ACTIVE_STATE); - EXPECT_CALL(*mock_sai_neighbor_api, remove_neighbor_entry) + EXPECT_CALL(*mock_sai_neighbor_api, remove_neighbor_entries) .WillOnce(Throw(logic_error("Mock logic error"))); SetMuxStateFromAppDb(STANDBY_STATE); EXPECT_EQ(ACTIVE_STATE, m_MuxCable->getState()); @@ -249,4 +277,12 @@ namespace mux_rollback_test SetMuxStateFromAppDb(STANDBY_STATE); EXPECT_EQ(ACTIVE_STATE, m_MuxCable->getState()); } + + TEST_F(MuxRollbackTest, StandbyToActiveNextHopTableFullRollbackToActive) + { + EXPECT_CALL(*mock_sai_next_hop_api, create_next_hop) + .WillOnce(Return(SAI_STATUS_TABLE_FULL)); + SetMuxStateFromAppDb(ACTIVE_STATE); + EXPECT_EQ(STANDBY_STATE, m_MuxCable->getState()); + } } diff --git a/tests/test_mux.py b/tests/test_mux.py index 9405312a5a..fce1b4f37c 100644 --- a/tests/test_mux.py +++ b/tests/test_mux.py @@ -101,6 +101,8 @@ class TestMuxTunnelBase(): DSCP_TO_TC_MAP = {str(i):str(1) for i in range(0, 64)} TC_TO_PRIORITY_GROUP_MAP = {str(i):str(i) for i in range(0, 8)} + BULK_NEIGHBOR_COUNT = 254 + def check_syslog(self, dvs, marker, err_log, expected_cnt): (exitcode, num) = dvs.runcmd(['sh', '-c', "awk \'/%s/,ENDFILE {print;}\' /var/log/syslog | grep \"%s\" | wc -l" % (marker, err_log)]) assert num.strip() >= str(expected_cnt) @@ -337,8 +339,66 @@ def del_route(self, dvs, route): ps = swsscommon.ProducerStateTable(apdb.db_connection, self.APP_ROUTE_TABLE) ps._del(route) + def wait_for_mux_state(self, dvs, interface, expected_state): + """ + Waits until state change completes - expected state is in state_db + """ + + apdb = dvs.get_app_db() + expected_field = {"state": expected_state} + apdb.wait_for_field_match(self.APP_MUX_CABLE, interface, expected_field) + + def bulk_neighbor_test(self, confdb, appdb, asicdb, dvs, dvs_route): + dvs.runcmd("ip neigh flush all") + self.add_fdb(dvs, "Ethernet0", "00-00-00-00-11-11") + self.set_mux_state(appdb, "Ethernet0", "active") + + class neighbor_info: + ipv4_key = "" + ipv6_key = "" + ipv4 = "" + ipv6 = "" + + def __init__(self, i): + self.ipv4 = "192.168.1." + str(i) + self.ipv6 = "fc02:1001::" + str(i) + + neighbor_list = [neighbor_info(i) for i in range(100, self.BULK_NEIGHBOR_COUNT)] + for neigh_info in neighbor_list: + self.add_neighbor(dvs, neigh_info.ipv4, "00:00:00:00:11:11") + self.add_neighbor(dvs, neigh_info.ipv6, "00:00:00:00:11:11") + neigh_info.ipv4_key = self.check_neigh_in_asic_db(asicdb, neigh_info.ipv4) + neigh_info.ipv6_key = self.check_neigh_in_asic_db(asicdb, neigh_info.ipv6) + + try: + self.set_mux_state(appdb, "Ethernet0", "standby") + self.wait_for_mux_state(dvs, "Ethernet0", "standby") + + for neigh_info in neighbor_list: + asicdb.wait_for_deleted_entry(self.ASIC_NEIGH_TABLE, neigh_info.ipv4_key) + asicdb.wait_for_deleted_entry(self.ASIC_NEIGH_TABLE, neigh_info.ipv6_key) + dvs_route.check_asicdb_route_entries( + [neigh_info.ipv4+self.IPV4_MASK, neigh_info.ipv6+self.IPV6_MASK] + ) + + self.set_mux_state(appdb, "Ethernet0", "active") + self.wait_for_mux_state(dvs, "Ethernet0", "active") + + for neigh_info in neighbor_list: + dvs_route.check_asicdb_deleted_route_entries( + [neigh_info.ipv4+self.IPV4_MASK, neigh_info.ipv6+self.IPV6_MASK] + ) + neigh_info.ipv4_key = self.check_neigh_in_asic_db(asicdb, neigh_info.ipv4) + neigh_info.ipv6_key = self.check_neigh_in_asic_db(asicdb, neigh_info.ipv6) + + finally: + for neigh_info in neighbor_list: + self.del_neighbor(dvs, neigh_info.ipv4) + self.del_neighbor(dvs, neigh_info.ipv6) + def create_and_test_neighbor(self, confdb, appdb, asicdb, dvs, dvs_route): + self.bulk_neighbor_test(confdb, appdb, asicdb, dvs, dvs_route) self.set_mux_state(appdb, "Ethernet0", "active") self.set_mux_state(appdb, "Ethernet4", "standby")