diff --git a/src/vma/dev/cq_mgr.cpp b/src/vma/dev/cq_mgr.cpp index e8390867a..189c9869b 100644 --- a/src/vma/dev/cq_mgr.cpp +++ b/src/vma/dev/cq_mgr.cpp @@ -580,17 +580,7 @@ void cq_mgr::reclaim_recv_buffer_helper(mem_buf_desc_t* buff) temp->p_next_desc = NULL; temp->p_prev_desc = NULL; temp->reset_ref_count(); - temp->rx.tcp.gro = 0; - temp->rx.is_vma_thr = false; - temp->rx.socketxtreme_polled = false; - temp->rx.flow_tag_id = 0; - temp->rx.tcp.p_ip_h = NULL; - temp->rx.tcp.p_tcp_h = NULL; - temp->rx.timestamps.sw.tv_nsec = 0; - temp->rx.timestamps.sw.tv_sec = 0; - temp->rx.timestamps.hw.tv_nsec = 0; - temp->rx.timestamps.hw.tv_sec = 0; - temp->rx.hw_raw_timestamp = 0; + memset(&temp->rx, 0, sizeof(temp->rx)); free_lwip_pbuf(&temp->lwip_pbuf); m_rx_pool.push_back(temp); } diff --git a/src/vma/dev/ring.h b/src/vma/dev/ring.h index f6d353c22..3663bed0f 100644 --- a/src/vma/dev/ring.h +++ b/src/vma/dev/ring.h @@ -88,6 +88,7 @@ class ring // Get/Release memory buffer descriptor with a linked data memory buffer virtual mem_buf_desc_t* mem_buf_tx_get(ring_user_id_t id, bool b_block, int n_num_mem_bufs = 1) = 0; virtual int mem_buf_tx_release(mem_buf_desc_t* p_mem_buf_desc_list, bool b_accounting, bool trylock = false) = 0; + virtual void mem_buf_rx_release(mem_buf_desc_t* p_mem_buf_desc) { buffer_pool::free_rx_lwip_pbuf_custom(&p_mem_buf_desc->lwip_pbuf.pbuf); }; virtual void send_ring_buffer(ring_user_id_t id, vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_attr attr) = 0; virtual void send_lwip_buffer(ring_user_id_t id, vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_attr attr) = 0; diff --git a/src/vma/dev/ring_bond.cpp b/src/vma/dev/ring_bond.cpp index 12455cba2..167c7de33 100644 --- a/src/vma/dev/ring_bond.cpp +++ b/src/vma/dev/ring_bond.cpp @@ -376,6 +376,21 @@ int ring_bond::mem_buf_tx_release(mem_buf_desc_t* p_mem_buf_desc_list, bool b_ac return ret; } +void ring_bond::mem_buf_rx_release(mem_buf_desc_t* p_mem_buf_desc) +{ + uint32_t i; + + for (i = 0; i < m_bond_rings.size(); i++) { + if (m_bond_rings[i] == p_mem_buf_desc->p_desc_owner) { + m_bond_rings[i]->mem_buf_rx_release(p_mem_buf_desc); + break; + } + } + if (i == m_bond_rings.size()) { + buffer_pool::free_rx_lwip_pbuf_custom(&p_mem_buf_desc->lwip_pbuf.pbuf); + } +} + void ring_bond::mem_buf_desc_return_single_to_owner_tx(mem_buf_desc_t* p_mem_buf_desc) { p_mem_buf_desc->p_desc_owner->mem_buf_desc_return_single_to_owner_tx(p_mem_buf_desc); diff --git a/src/vma/dev/ring_bond.h b/src/vma/dev/ring_bond.h index b643346fc..b3579ed30 100644 --- a/src/vma/dev/ring_bond.h +++ b/src/vma/dev/ring_bond.h @@ -59,6 +59,7 @@ class ring_bond : public ring { virtual void adapt_cq_moderation(); virtual bool reclaim_recv_buffers(descq_t *rx_reuse); virtual bool reclaim_recv_buffers(mem_buf_desc_t* rx_reuse_lst); + virtual void mem_buf_rx_release(mem_buf_desc_t* p_mem_buf_desc); virtual int drain_and_proccess(); virtual int wait_for_notification_and_process_element(int cq_channel_fd, uint64_t* p_cq_poll_sn, void* pv_fd_ready_array = NULL); virtual int get_num_resources() const { return m_bond_rings.size(); }; diff --git a/src/vma/dev/ring_simple.cpp b/src/vma/dev/ring_simple.cpp index 06d64abdf..e5094b955 100644 --- a/src/vma/dev/ring_simple.cpp +++ b/src/vma/dev/ring_simple.cpp @@ -629,6 +629,12 @@ int ring_simple::mem_buf_tx_release(mem_buf_desc_t* p_mem_buf_desc_list, bool b_ return accounting; } +void ring_simple::mem_buf_rx_release(mem_buf_desc_t* p_mem_buf_desc) +{ + p_mem_buf_desc->p_next_desc = NULL; + reclaim_recv_buffers(p_mem_buf_desc); +} + /* note that this function is inline, so keep it above the functions using it */ inline int ring_simple::send_buffer(vma_ibv_send_wr* p_send_wqe, vma_wr_tx_packet_attr attr) { diff --git a/src/vma/dev/ring_simple.h b/src/vma/dev/ring_simple.h index cd3266a0a..151e937fd 100644 --- a/src/vma/dev/ring_simple.h +++ b/src/vma/dev/ring_simple.h @@ -69,6 +69,7 @@ class ring_simple : public ring_slave virtual bool reclaim_recv_buffers(mem_buf_desc_t* rx_reuse_lst); bool reclaim_recv_buffers_no_lock(mem_buf_desc_t* rx_reuse_lst); // No locks virtual int reclaim_recv_single_buffer(mem_buf_desc_t* rx_reuse); // No locks + virtual void mem_buf_rx_release(mem_buf_desc_t* p_mem_buf_desc); virtual int socketxtreme_poll(struct vma_completion_t *vma_completions, unsigned int ncompletions, int flags); virtual int drain_and_proccess(); virtual int wait_for_notification_and_process_element(int cq_channel_fd, uint64_t* p_cq_poll_sn, void* pv_fd_ready_array = NULL); diff --git a/src/vma/dev/ring_tap.cpp b/src/vma/dev/ring_tap.cpp index 19ec7bc12..ab63b73f4 100644 --- a/src/vma/dev/ring_tap.cpp +++ b/src/vma/dev/ring_tap.cpp @@ -319,17 +319,7 @@ bool ring_tap::reclaim_recv_buffers(mem_buf_desc_t *buff) temp->p_next_desc = NULL; temp->p_prev_desc = NULL; temp->reset_ref_count(); - temp->rx.tcp.gro = 0; - temp->rx.is_vma_thr = false; - temp->rx.socketxtreme_polled = false; - temp->rx.flow_tag_id = 0; - temp->rx.tcp.p_ip_h = NULL; - temp->rx.tcp.p_tcp_h = NULL; - temp->rx.timestamps.sw.tv_nsec = 0; - temp->rx.timestamps.sw.tv_sec = 0; - temp->rx.timestamps.hw.tv_nsec = 0; - temp->rx.timestamps.hw.tv_sec = 0; - temp->rx.hw_raw_timestamp = 0; + memset(&temp->rx, 0, sizeof(temp->rx)); free_lwip_pbuf(&temp->lwip_pbuf); m_rx_pool.push_back(temp); } diff --git a/src/vma/sock/sockinfo_tcp.cpp b/src/vma/sock/sockinfo_tcp.cpp index 2141f78f7..d9bf7aaed 100644 --- a/src/vma/sock/sockinfo_tcp.cpp +++ b/src/vma/sock/sockinfo_tcp.cpp @@ -143,6 +143,8 @@ inline void sockinfo_tcp::init_pbuf_custom(mem_buf_desc_t *p_desc) p_desc->lwip_pbuf.pbuf.type = PBUF_REF; p_desc->lwip_pbuf.pbuf.next = NULL; p_desc->lwip_pbuf.pbuf.payload = (u8_t *)p_desc->p_buffer + p_desc->rx.tcp.n_transport_header_len; + /* Override default free function to return rx pbuf to the CQ cache */ + p_desc->lwip_pbuf.custom_free_function = sockinfo_tcp::tcp_rx_pbuf_free; } /* change default rx_wait impl to flow based one */ @@ -4527,6 +4529,16 @@ struct pbuf * sockinfo_tcp::tcp_tx_pbuf_alloc(void* p_conn) return (struct pbuf *)p_desc; } +void sockinfo_tcp::tcp_rx_pbuf_free(struct pbuf *p_buff) +{ + mem_buf_desc_t *desc = (mem_buf_desc_t *)p_buff; + + if (desc->p_desc_owner != NULL) + desc->p_desc_owner->mem_buf_rx_release(desc); + else + buffer_pool::free_rx_lwip_pbuf_custom(p_buff); +} + //single buffer only void sockinfo_tcp::tcp_tx_pbuf_free(void* p_conn, struct pbuf *p_buff) { diff --git a/src/vma/sock/sockinfo_tcp.h b/src/vma/sock/sockinfo_tcp.h index 013107178..20f4111da 100644 --- a/src/vma/sock/sockinfo_tcp.h +++ b/src/vma/sock/sockinfo_tcp.h @@ -182,6 +182,7 @@ class sockinfo_tcp : public sockinfo, public timer_handler static struct pbuf * tcp_tx_pbuf_alloc(void* p_conn); static void tcp_tx_pbuf_free(void* p_conn, struct pbuf *p_buff); + static void tcp_rx_pbuf_free(struct pbuf *p_buff); static struct tcp_seg * tcp_seg_alloc(void* p_conn); static void tcp_seg_free(void* p_conn, struct tcp_seg * seg);