Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DOCA TX/RX statistics and leftovers #261

Open
wants to merge 7 commits into
base: doca_xlio_vNext
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/core/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ libxlio_la_SOURCES := \
dev/ring_bond.cpp \
dev/ring_slave.cpp \
dev/ring_simple.cpp \
dev/ring_tap.cpp \
dev/ring_allocation_logic.cpp \
\
event/delta_timer.cpp \
Expand Down Expand Up @@ -202,7 +201,6 @@ libxlio_la_SOURCES := \
dev/ring_bond.h \
dev/ring_slave.h \
dev/ring_simple.h \
dev/ring_tap.h \
dev/ring_allocation_logic.h \
dev/wqe_send_handler.h \
dev/xlio_ti.h \
Expand Down
63 changes: 29 additions & 34 deletions src/core/dev/cq_mgr_rx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,10 @@ atomic_t cq_mgr_rx::m_n_cq_id_counter_rx = ATOMIC_INIT(1);

uint64_t cq_mgr_rx::m_n_global_sn_rx = 0;

cq_mgr_rx::cq_mgr_rx(ring_simple *p_ring, ib_ctx_handler *p_ib_ctx_handler, int cq_size,
struct ibv_comp_channel *p_comp_event_channel)
: m_p_ring(p_ring)
cq_mgr_rx::cq_mgr_rx(ring_simple *p_ring, hw_queue_rx *hqrx_ptr, ib_ctx_handler *p_ib_ctx_handler,
int cq_size, struct ibv_comp_channel *p_comp_event_channel)
: m_hqrx_ptr(hqrx_ptr)
, m_p_ring(p_ring)
, m_n_sysvar_cq_poll_batch_max(safe_mce_sys().cq_poll_batch_max)
, m_n_sysvar_progress_engine_wce_max(safe_mce_sys().progress_engine_wce_max)
, m_p_cq_stat(&m_cq_stat_static) // use local copy of stats by default
Expand Down Expand Up @@ -151,7 +152,7 @@ cq_mgr_rx::~cq_mgr_rx()
m_p_cq_stat->n_rx_sw_queue_len = m_rx_queue.size();

g_buffer_pool_rx_rwqe->put_buffers_thread_safe(&m_rx_pool, m_rx_pool.size());
m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
}

cq_logfunc("destroying ibv_cq");
Expand All @@ -170,19 +171,21 @@ cq_mgr_rx::~cq_mgr_rx()

void cq_mgr_rx::statistics_print()
{
if (m_p_cq_stat->n_rx_pkt_drop || m_p_cq_stat->n_rx_sw_queue_len ||
m_p_cq_stat->n_rx_drained_at_once_max || m_p_cq_stat->n_buffer_pool_len) {
if (m_hqrx_ptr &&
(m_p_cq_stat->n_rx_pkt_drop || m_p_cq_stat->n_rx_sw_queue_len ||
m_hqrx_ptr->m_hwq_rx_stats.n_rx_drained_at_once_max ||
m_hqrx_ptr->m_hwq_rx_stats.n_rx_buffer_pool_len)) {
cq_logdbg_no_funcname("Packets dropped: %12llu",
(unsigned long long int)m_p_cq_stat->n_rx_pkt_drop);
cq_logdbg_no_funcname("Drained max: %17u", m_p_cq_stat->n_rx_drained_at_once_max);
cq_logdbg_no_funcname("Drained max: %17u",
m_hqrx_ptr->m_hwq_rx_stats.n_rx_drained_at_once_max);
cq_logdbg_no_funcname("CQE errors: %18llu",
(unsigned long long int)m_p_cq_stat->n_rx_cqe_error);
(unsigned long long int)m_hqrx_ptr->m_hwq_rx_stats.n_rx_task_error);
}
}

void cq_mgr_rx::add_hqrx(hw_queue_rx *hqrx_ptr)
void cq_mgr_rx::add_hqrx()
{
m_hqrx_ptr = hqrx_ptr;
m_hqrx_ptr->m_rq_wqe_counter = 0; // In case of bonded hqrx, wqe_counter must be reset to zero
m_rx_hot_buffer = nullptr;

Expand All @@ -191,19 +194,17 @@ void cq_mgr_rx::add_hqrx(hw_queue_rx *hqrx_ptr)
}

VALGRIND_MAKE_MEM_DEFINED(&m_mlx5_cq, sizeof(m_mlx5_cq));
cq_logfunc("hqrx_ptr=%p m_mlx5_cq.dbrec=%p m_mlx5_cq.cq_buf=%p", hqrx_ptr, m_mlx5_cq.dbrec,
cq_logfunc("hqrx_ptr=%p m_mlx5_cq.dbrec=%p m_mlx5_cq.cq_buf=%p", m_hqrx_ptr, m_mlx5_cq.dbrec,
m_mlx5_cq.cq_buf);

descq_t temp_desc_list;
temp_desc_list.set_id("cq_mgr_rx (%p) : temp_desc_list", this);

m_p_cq_stat->n_rx_drained_at_once_max = 0;

/* return_extra_buffers(); */ // todo??

// Initial fill of receiver work requests
uint32_t hqrx_wr_num = hqrx_ptr->get_rx_max_wr_num();
cq_logdbg("Trying to push %d WRE to allocated hqrx (%p)", hqrx_wr_num, hqrx_ptr);
uint32_t hqrx_wr_num = m_hqrx_ptr->get_rx_max_wr_num();
cq_logdbg("Trying to push %d WRE to allocated hqrx (%p)", hqrx_wr_num, m_hqrx_ptr);
while (hqrx_wr_num) {
uint32_t n_num_mem_bufs = m_n_sysvar_rx_num_wr_to_post_recv;
if (n_num_mem_bufs > hqrx_wr_num) {
Expand All @@ -215,39 +216,31 @@ void cq_mgr_rx::add_hqrx(hw_queue_rx *hqrx_ptr)
VLOG_PRINTF_INFO_ONCE_THEN_ALWAYS(
VLOG_WARNING, VLOG_DEBUG,
"Out of mem_buf_desc in global RX buffer pool for hqrx initialization (hqrx=%p)",
hqrx_ptr);
m_hqrx_ptr);
break;
}

hqrx_ptr->post_recv_buffers(&temp_desc_list, temp_desc_list.size());
m_hqrx_ptr->post_recv_buffers(&temp_desc_list, temp_desc_list.size());
if (!temp_desc_list.empty()) {
cq_logdbg("hqrx_ptr post recv is already full (push=%d, planned=%d)",
hqrx_ptr->get_rx_max_wr_num() - hqrx_wr_num, hqrx_ptr->get_rx_max_wr_num());
m_hqrx_ptr->get_rx_max_wr_num() - hqrx_wr_num,
m_hqrx_ptr->get_rx_max_wr_num());
g_buffer_pool_rx_rwqe->put_buffers_thread_safe(&temp_desc_list, temp_desc_list.size());
break;
}
hqrx_wr_num -= n_num_mem_bufs;
}

cq_logdbg("Successfully post_recv hqrx with %d new Rx buffers (planned=%d)",
hqrx_ptr->get_rx_max_wr_num() - hqrx_wr_num, hqrx_ptr->get_rx_max_wr_num());
m_hqrx_ptr->get_rx_max_wr_num() - hqrx_wr_num, m_hqrx_ptr->get_rx_max_wr_num());

m_debt = 0;
}

void cq_mgr_rx::del_hqrx(hw_queue_rx *hqrx_ptr)
void cq_mgr_rx::del_hqrx()
{
BULLSEYE_EXCLUDE_BLOCK_START
if (m_hqrx_ptr != hqrx_ptr) {
cq_logdbg("wrong hqrx_ptr=%p != m_hqrx_ptr=%p", hqrx_ptr, m_hqrx_ptr);
return;
}
BULLSEYE_EXCLUDE_BLOCK_END
cq_logdbg("m_hqrx_ptr=%p", m_hqrx_ptr);
return_extra_buffers();

clean_cq();
m_hqrx_ptr = nullptr;
m_debt = 0;
}

Expand Down Expand Up @@ -318,7 +311,7 @@ bool cq_mgr_rx::request_more_buffers()
return false;
};

m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
return true;
}

Expand All @@ -331,7 +324,7 @@ void cq_mgr_rx::return_extra_buffers()

cq_logfunc("releasing %d buffers to global rx pool", buff_to_rel);
g_buffer_pool_rx_rwqe->put_buffers_thread_safe(&m_rx_pool, buff_to_rel);
m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
}

mem_buf_desc_t *cq_mgr_rx::cqe_process_rx(mem_buf_desc_t *p_mem_buf_desc, enum buff_status_e status)
Expand Down Expand Up @@ -367,7 +360,7 @@ bool cq_mgr_rx::compensate_qp_poll_success(mem_buf_desc_t *buff_cur)
size_t buffers = std::min<size_t>(m_debt, m_rx_pool.size());
m_hqrx_ptr->post_recv_buffers(&m_rx_pool, buffers);
m_debt -= buffers;
m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
} else if (m_b_sysvar_cq_keep_qp_full || m_debt >= (int)m_hqrx_ptr->m_rx_num_wr) {
m_p_cq_stat->n_rx_pkt_drop++;
m_hqrx_ptr->post_recv_buffer(buff_cur);
Expand All @@ -386,8 +379,8 @@ void cq_mgr_rx::compensate_qp_poll_failed()
if (likely(m_rx_pool.size() || request_more_buffers())) {
size_t buffers = std::min<size_t>(m_debt, m_rx_pool.size());
m_hqrx_ptr->post_recv_buffers(&m_rx_pool, buffers);
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
m_debt -= buffers;
m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
}
}
}
Expand All @@ -409,7 +402,7 @@ void cq_mgr_rx::reclaim_recv_buffer_helper(mem_buf_desc_t *buff)
free_lwip_pbuf(&temp->lwip_pbuf);
m_rx_pool.push_back(temp);
}
m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
} else {
cq_logfunc("Buffer returned to wrong CQ");
g_buffer_pool_rx_rwqe->put_buffers_thread_safe(buff);
Expand Down Expand Up @@ -472,6 +465,7 @@ bool cq_mgr_rx::request_notification()
else
{
m_b_notification_armed = true;
++m_hqrx_ptr->m_hwq_rx_stats.n_rx_interrupt_requests;
}
ENDIF_VERBS_FAILURE;
}
Expand Down Expand Up @@ -508,6 +502,7 @@ void cq_mgr_rx::wait_for_notification()

// Clear flag
m_b_notification_armed = false;
++m_hqrx_ptr->m_hwq_rx_stats.n_rx_interrupt_received;
}
ENDIF_VERBS_FAILURE;
} else {
Expand Down
8 changes: 4 additions & 4 deletions src/core/dev/cq_mgr_rx.h
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ class cq_mgr_rx {
BS_GENERAL_ERR
};

cq_mgr_rx(ring_simple *p_ring, ib_ctx_handler *p_ib_ctx_handler, int cq_size,
struct ibv_comp_channel *p_comp_event_channel);
cq_mgr_rx(ring_simple *p_ring, hw_queue_rx *hqrx_ptr, ib_ctx_handler *p_ib_ctx_handler,
int cq_size, struct ibv_comp_channel *p_comp_event_channel);
virtual ~cq_mgr_rx();

void configure(int cq_size);
Expand Down Expand Up @@ -124,8 +124,8 @@ class cq_mgr_rx {
void mem_buf_desc_return_to_owner(mem_buf_desc_t *p_mem_buf_desc,
void *pv_fd_ready_array = nullptr);

virtual void add_hqrx(hw_queue_rx *hqrx_ptr);
virtual void del_hqrx(hw_queue_rx *hqrx_ptr);
virtual void add_hqrx();
virtual void del_hqrx();

virtual uint32_t clean_cq() = 0;

Expand Down
2 changes: 2 additions & 0 deletions src/core/dev/cq_mgr_rx_inl.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@
inline void cq_mgr_rx::process_recv_buffer(mem_buf_desc_t *p_mem_buf_desc, void *pv_fd_ready_array)
{
// Assume locked!!!
m_hqrx_ptr->m_hwq_rx_stats.n_rx_byte_count += p_mem_buf_desc->sz_data;
++m_hqrx_ptr->m_hwq_rx_stats.n_rx_pkt_count;

// Pass the Rx buffer ib_comm_mgr for further IP processing
if (!m_p_ring->rx_process_buffer(p_mem_buf_desc, pv_fd_ready_array)) {
Expand Down
21 changes: 9 additions & 12 deletions src/core/dev/cq_mgr_rx_regrq.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,10 @@
#define cq_logpanic __log_info_panic
#define cq_logfuncall __log_info_funcall

cq_mgr_rx_regrq::cq_mgr_rx_regrq(ring_simple *p_ring, ib_ctx_handler *p_ib_ctx_handler,
uint32_t cq_size, struct ibv_comp_channel *p_comp_event_channel)
: cq_mgr_rx(p_ring, p_ib_ctx_handler, cq_size, p_comp_event_channel)
cq_mgr_rx_regrq::cq_mgr_rx_regrq(ring_simple *p_ring, hw_queue_rx *hqrx_ptr,
ib_ctx_handler *p_ib_ctx_handler, uint32_t cq_size,
struct ibv_comp_channel *p_comp_event_channel)
: cq_mgr_rx(p_ring, hqrx_ptr, p_ib_ctx_handler, cq_size, p_comp_event_channel)
{
cq_logfunc("");
}
Expand All @@ -65,10 +66,6 @@ uint32_t cq_mgr_rx_regrq::clean_cq()
uint64_t cq_poll_sn = 0;
mem_buf_desc_t *buff;

if (!m_hqrx_ptr) { // Sanity check
return 0;
}

buff_status_e status = BS_OK;
while ((buff = poll(status))) {
if (cqe_process_rx(buff, status)) {
Expand Down Expand Up @@ -153,8 +150,8 @@ void cq_mgr_rx_regrq::cqe_to_mem_buff_desc(struct xlio_mlx5_cqe *cqe,
(cqe->hds_ip_ext & MLX5_CQE_L3_OK));
if (cqe->lro_num_seg > 1) {
lro_update_hdr(cqe, p_rx_wc_buf_desc);
m_p_cq_stat->n_rx_lro_packets++;
m_p_cq_stat->n_rx_lro_bytes += p_rx_wc_buf_desc->sz_data;
m_hqrx_ptr->m_hwq_rx_stats.n_rx_lro_packets++;
m_hqrx_ptr->m_hwq_rx_stats.n_rx_lro_bytes += p_rx_wc_buf_desc->sz_data;
}
return;
}
Expand Down Expand Up @@ -199,7 +196,7 @@ void cq_mgr_rx_regrq::cqe_to_mem_buff_desc(struct xlio_mlx5_cqe *cqe,
case MLX5_CQE_INVALID:
case MLX5_CQE_REQ_ERR:
case MLX5_CQE_RESP_ERR:
m_p_cq_stat->n_rx_cqe_error++;
m_hqrx_ptr->m_hwq_rx_stats.n_rx_task_error++;
break;
}
}
Expand Down Expand Up @@ -268,8 +265,8 @@ int cq_mgr_rx_regrq::drain_and_proccess(uintptr_t *p_recycle_buffers_last_wr_id

// Update cq statistics
m_p_cq_stat->n_rx_sw_queue_len = m_rx_queue.size();
m_p_cq_stat->n_rx_drained_at_once_max =
std::max(ret_total, m_p_cq_stat->n_rx_drained_at_once_max);
m_hqrx_ptr->m_hwq_rx_stats.n_rx_drained_at_once_max =
std::max(ret_total, m_hqrx_ptr->m_hwq_rx_stats.n_rx_drained_at_once_max);

return ret_total;
}
Expand Down
4 changes: 2 additions & 2 deletions src/core/dev/cq_mgr_rx_regrq.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@

class cq_mgr_rx_regrq : public cq_mgr_rx {
public:
cq_mgr_rx_regrq(ring_simple *p_ring, ib_ctx_handler *p_ib_ctx_handler, uint32_t cq_size,
struct ibv_comp_channel *p_comp_event_channel);
cq_mgr_rx_regrq(ring_simple *p_ring, hw_queue_rx *hqrx_ptr, ib_ctx_handler *p_ib_ctx_handler,
uint32_t cq_size, struct ibv_comp_channel *p_comp_event_channel);

virtual ~cq_mgr_rx_regrq() override;

Expand Down
30 changes: 14 additions & 16 deletions src/core/dev/cq_mgr_rx_strq.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,11 @@
##log_args); \
} while (0)

cq_mgr_rx_strq::cq_mgr_rx_strq(ring_simple *p_ring, ib_ctx_handler *p_ib_ctx_handler,
uint32_t cq_size, uint32_t stride_size_bytes, uint32_t strides_num,
cq_mgr_rx_strq::cq_mgr_rx_strq(ring_simple *p_ring, hw_queue_rx *hqrx_ptr,
ib_ctx_handler *p_ib_ctx_handler, uint32_t cq_size,
uint32_t stride_size_bytes, uint32_t strides_num,
struct ibv_comp_channel *p_comp_event_channel)
: cq_mgr_rx(p_ring, p_ib_ctx_handler, cq_size, p_comp_event_channel)
: cq_mgr_rx(p_ring, hqrx_ptr, p_ib_ctx_handler, cq_size, p_comp_event_channel)
, _owner_ring(p_ring)
, _stride_size_bytes(stride_size_bytes)
, _strides_num(strides_num)
Expand Down Expand Up @@ -127,10 +128,6 @@ uint32_t cq_mgr_rx_strq::clean_cq()
uint32_t ret_total = 0;
uint64_t cq_poll_sn = 0;

if (!m_hqrx_ptr) { // Sanity check
return 0;
}

mem_buf_desc_t *stride_buf = nullptr;
buff_status_e status = BS_OK;
while (poll(status, stride_buf) || stride_buf) {
Expand Down Expand Up @@ -257,8 +254,8 @@ inline bool cq_mgr_rx_strq::strq_cqe_to_mem_buff_desc(struct xlio_mlx5_cqe *cqe,
#endif /* DEFINED_UTLS */
if (cqe->lro_num_seg > 1) {
lro_update_hdr(cqe, _hot_buffer_stride);
m_p_cq_stat->n_rx_lro_packets++;
m_p_cq_stat->n_rx_lro_bytes += _hot_buffer_stride->sz_data;
m_hqrx_ptr->m_hwq_rx_stats.n_rx_lro_packets++;
m_hqrx_ptr->m_hwq_rx_stats.n_rx_lro_bytes += _hot_buffer_stride->sz_data;
}
break;
}
Expand Down Expand Up @@ -402,8 +399,8 @@ int cq_mgr_rx_strq::drain_and_proccess(uintptr_t *p_recycle_buffers_last_wr_id)

// Update cq statistics
m_p_cq_stat->n_rx_sw_queue_len = m_rx_queue.size();
m_p_cq_stat->n_rx_drained_at_once_max =
std::max(ret_total, m_p_cq_stat->n_rx_drained_at_once_max);
m_hqrx_ptr->m_hwq_rx_stats.n_rx_drained_at_once_max =
std::max(ret_total, m_hqrx_ptr->m_hwq_rx_stats.n_rx_drained_at_once_max);

return ret_total;
}
Expand Down Expand Up @@ -471,12 +468,12 @@ bool cq_mgr_rx_strq::poll_and_process_element_rx(void *pv_fd_ready_array)
return (rx_polled < m_n_sysvar_cq_poll_batch_max);
}

void cq_mgr_rx_strq::add_hqrx(hw_queue_rx *hqrx)
void cq_mgr_rx_strq::add_hqrx()
{
cq_logfunc("");
_hot_buffer_stride = nullptr;
_current_wqe_consumed_bytes = 0U;
cq_mgr_rx::add_hqrx(hqrx);
cq_mgr_rx::add_hqrx();
}

void cq_mgr_rx_strq::statistics_print()
Expand All @@ -487,8 +484,9 @@ void cq_mgr_rx_strq::statistics_print()
cq_logdbg_no_funcname("Max Strides per Packet: %12" PRIu16,
m_p_cq_stat->n_rx_max_stirde_per_packet);
cq_logdbg_no_funcname("Strides count: %12" PRIu64, m_p_cq_stat->n_rx_stride_count);
cq_logdbg_no_funcname("LRO packet count: %12" PRIu64, m_p_cq_stat->n_rx_lro_packets);
cq_logdbg_no_funcname("LRO bytes: %12" PRIu64, m_p_cq_stat->n_rx_lro_bytes);
cq_logdbg_no_funcname("LRO packet count: %12" PRIu64,
m_hqrx_ptr->m_hwq_rx_stats.n_rx_lro_packets);
cq_logdbg_no_funcname("LRO bytes: %12" PRIu64, m_hqrx_ptr->m_hwq_rx_stats.n_rx_lro_bytes);
}

void cq_mgr_rx_strq::reclaim_recv_buffer_helper(mem_buf_desc_t *buff)
Expand Down Expand Up @@ -523,7 +521,7 @@ void cq_mgr_rx_strq::reclaim_recv_buffer_helper(mem_buf_desc_t *buff)
return_stride(temp);
}

m_p_cq_stat->n_buffer_pool_len = m_rx_pool.size();
m_hqrx_ptr->update_rx_buffer_pool_len_stats();
} else {
cq_logfunc("Stride returned to wrong CQ");
g_buffer_pool_rx_ptr->put_buffers_thread_safe(buff);
Expand Down
6 changes: 3 additions & 3 deletions src/core/dev/cq_mgr_rx_strq.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@

class cq_mgr_rx_strq : public cq_mgr_rx {
public:
cq_mgr_rx_strq(ring_simple *p_ring, ib_ctx_handler *p_ib_ctx_handler, uint32_t cq_size,
uint32_t stride_size_bytes, uint32_t strides_num,
cq_mgr_rx_strq(ring_simple *p_ring, hw_queue_rx *hqrx_ptr, ib_ctx_handler *p_ib_ctx_handler,
uint32_t cq_size, uint32_t stride_size_bytes, uint32_t strides_num,
struct ibv_comp_channel *p_comp_event_channel);

virtual ~cq_mgr_rx_strq() override;

virtual int drain_and_proccess(uintptr_t *p_recycle_buffers_last_wr_id = NULL) override;
virtual bool poll_and_process_element_rx(void *pv_fd_ready_array = NULL) override;
virtual void add_hqrx(hw_queue_rx *qp) override;
virtual void add_hqrx() override;
virtual uint32_t clean_cq() override;

protected:
Expand Down
Loading