diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c index 2abdee189dc..85068fa91c6 100644 --- a/prov/efa/src/efa_base_ep.c +++ b/prov/efa/src/efa_base_ep.c @@ -4,6 +4,7 @@ #include #include "efa.h" #include "efa_av.h" +#include "efa_cq.h" #include "rdm/efa_rdm_protocol.h" int efa_base_ep_bind_av(struct efa_base_ep *base_ep, struct efa_av *av) @@ -520,3 +521,13 @@ const char *efa_base_ep_get_peer_raw_addr_str(struct efa_base_ep *base_ep, fi_ad { return ofi_straddr(buf, buflen, FI_ADDR_EFA, efa_base_ep_get_peer_raw_addr(base_ep, addr)); } + +struct efa_cq *efa_base_ep_get_tx_cq(struct efa_base_ep *ep) +{ + return ep->util_ep.tx_cq ? container_of(ep->util_ep.tx_cq, struct efa_cq, util_cq) : NULL; +} + +struct efa_cq *efa_base_ep_get_rx_cq(struct efa_base_ep *ep) +{ + return ep->util_ep.rx_cq ? container_of(ep->util_ep.rx_cq, struct efa_cq, util_cq) : NULL; +} diff --git a/prov/efa/src/efa_base_ep.h b/prov/efa/src/efa_base_ep.h index 86657c5dc12..a7d1526919e 100644 --- a/prov/efa/src/efa_base_ep.h +++ b/prov/efa/src/efa_base_ep.h @@ -109,4 +109,9 @@ struct efa_ep_addr *efa_base_ep_get_peer_raw_addr(struct efa_base_ep *base_ep, const char *efa_base_ep_get_peer_raw_addr_str(struct efa_base_ep *base_ep, fi_addr_t addr, char *buf, size_t *buflen); + +struct efa_cq *efa_base_ep_get_tx_cq(struct efa_base_ep *ep); + +struct efa_cq *efa_base_ep_get_rx_cq(struct efa_base_ep *ep); + #endif diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c index 5a18ef17003..24051cc2e8a 100644 --- a/prov/efa/src/rdm/efa_rdm_cq.c +++ b/prov/efa/src/rdm/efa_rdm_cq.c @@ -36,16 +36,16 @@ int efa_rdm_cq_close(struct fid *fid) retv = 0; - cq = container_of(fid, struct efa_rdm_cq, util_cq.cq_fid.fid); + cq = container_of(fid, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); - if (cq->ibv_cq.ibv_cq_ex) { - ret = -ibv_destroy_cq(ibv_cq_ex_to_cq(cq->ibv_cq.ibv_cq_ex)); + if (cq->efa_cq.ibv_cq.ibv_cq_ex) { + ret = -ibv_destroy_cq(ibv_cq_ex_to_cq(cq->efa_cq.ibv_cq.ibv_cq_ex)); if (ret) { EFA_WARN(FI_LOG_CQ, "Unable to close ibv cq: %s\n", fi_strerror(-ret)); return ret; } - cq->ibv_cq.ibv_cq_ex = NULL; + cq->efa_cq.ibv_cq.ibv_cq_ex = NULL; } if (cq->shm_cq) { @@ -56,7 +56,7 @@ int efa_rdm_cq_close(struct fid *fid) } } - ret = ofi_cq_cleanup(&cq->util_cq); + ret = ofi_cq_cleanup(&cq->efa_cq.util_cq); if (ret) return ret; free(cq); @@ -435,13 +435,13 @@ void efa_rdm_cq_poll_ibv_cq(ssize_t cqe_to_process, struct efa_ibv_cq *ibv_cq) int prov_errno; struct efa_rdm_ep *ep = NULL; struct fi_cq_err_entry err_entry; - struct efa_rdm_cq *efa_rdm_cq; + struct efa_cq *efa_cq; struct efa_domain *efa_domain; struct efa_qp *qp; struct dlist_entry rx_progressed_ep_list, *tmp; - efa_rdm_cq = container_of(ibv_cq, struct efa_rdm_cq, ibv_cq); - efa_domain = container_of(efa_rdm_cq->util_cq.domain, struct efa_domain, util_domain); + efa_cq = container_of(ibv_cq, struct efa_cq, ibv_cq); + efa_domain = container_of(efa_cq->util_cq.domain, struct efa_domain, util_domain); dlist_init(&rx_progressed_ep_list); /* Call ibv_start_poll only once */ @@ -538,7 +538,7 @@ void efa_rdm_cq_poll_ibv_cq(ssize_t cqe_to_process, struct efa_ibv_cq *ibv_cq) .prov_errno = prov_errno, .op_context = NULL }; - ofi_cq_write_error(&efa_rdm_cq->util_cq, &err_entry); + ofi_cq_write_error(&efa_cq->util_cq, &err_entry); } if (should_end_poll) @@ -559,9 +559,9 @@ static ssize_t efa_rdm_cq_readfrom(struct fid_cq *cq_fid, void *buf, size_t coun ssize_t ret; struct efa_domain *domain; - cq = container_of(cq_fid, struct efa_rdm_cq, util_cq.cq_fid.fid); + cq = container_of(cq_fid, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); - domain = container_of(cq->util_cq.domain, struct efa_domain, util_domain); + domain = container_of(cq->efa_cq.util_cq.domain, struct efa_domain, util_domain); ofi_genlock_lock(&domain->srx_lock); @@ -573,13 +573,13 @@ static ssize_t efa_rdm_cq_readfrom(struct fid_cq *cq_fid, void *buf, size_t coun * completion to efa. Use ofi_cq_read_entries to get the number of * shm completions without progressing efa ep again. */ - ret = ofi_cq_read_entries(&cq->util_cq, buf, count, src_addr); + ret = ofi_cq_read_entries(&cq->efa_cq.util_cq, buf, count, src_addr); if (ret > 0) goto out; } - ret = ofi_cq_readfrom(&cq->util_cq.cq_fid, buf, count, src_addr); + ret = ofi_cq_readfrom(&cq->efa_cq.util_cq.cq_fid, buf, count, src_addr); out: ofi_genlock_unlock(&domain->srx_lock); @@ -608,8 +608,8 @@ static void efa_rdm_cq_progress(struct util_cq *cq) struct fid_list_entry *fid_entry; ofi_genlock_lock(&cq->ep_list_lock); - efa_rdm_cq = container_of(cq, struct efa_rdm_cq, util_cq); - efa_domain = container_of(efa_rdm_cq->util_cq.domain, struct efa_domain, util_domain); + efa_rdm_cq = container_of(cq, struct efa_rdm_cq, efa_cq.util_cq); + efa_domain = container_of(efa_rdm_cq->efa_cq.util_cq.domain, struct efa_domain, util_domain); /** * TODO: It's better to just post the initial batch of internal rx pkts during ep enable @@ -671,19 +671,19 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, dlist_init(&cq->ibv_cq_poll_list); cq->need_to_scan_ep_list = false; - ret = ofi_cq_init(&efa_prov, domain, attr, &cq->util_cq, + ret = ofi_cq_init(&efa_prov, domain, attr, &cq->efa_cq.util_cq, &efa_rdm_cq_progress, context); if (ret) goto free; - ret = efa_cq_ibv_cq_ex_open(attr, efa_domain->device->ibv_ctx, &cq->ibv_cq.ibv_cq_ex, &cq->ibv_cq.ibv_cq_ex_type); + ret = efa_cq_ibv_cq_ex_open(attr, efa_domain->device->ibv_ctx, &cq->efa_cq.ibv_cq.ibv_cq_ex, &cq->efa_cq.ibv_cq.ibv_cq_ex_type); if (ret) { EFA_WARN(FI_LOG_CQ, "Unable to create extended CQ: %s\n", fi_strerror(ret)); goto close_util_cq; } - *cq_fid = &cq->util_cq.cq_fid; + *cq_fid = &cq->efa_cq.util_cq.cq_fid; (*cq_fid)->fid.ops = &efa_rdm_cq_fi_ops; (*cq_fid)->ops = &efa_rdm_cq_ops; @@ -693,7 +693,7 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, /* Bind ep with shm provider's cq */ shm_cq_attr.flags |= FI_PEER; peer_cq_context.size = sizeof(peer_cq_context); - peer_cq_context.cq = cq->util_cq.peer_cq; + peer_cq_context.cq = cq->efa_cq.util_cq.peer_cq; ret = fi_cq_open(efa_domain->shm_domain, &shm_cq_attr, &cq->shm_cq, &peer_cq_context); if (ret) { @@ -704,12 +704,12 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, return 0; destroy_ibv_cq: - retv = -ibv_destroy_cq(ibv_cq_ex_to_cq(cq->ibv_cq.ibv_cq_ex)); + retv = -ibv_destroy_cq(ibv_cq_ex_to_cq(cq->efa_cq.ibv_cq.ibv_cq_ex)); if (retv) EFA_WARN(FI_LOG_CQ, "Unable to close ibv cq: %s\n", fi_strerror(-retv)); close_util_cq: - retv = ofi_cq_cleanup(&cq->util_cq); + retv = ofi_cq_cleanup(&cq->efa_cq.util_cq); if (retv) EFA_WARN(FI_LOG_CQ, "Unable to close util cq: %s\n", fi_strerror(-retv)); diff --git a/prov/efa/src/rdm/efa_rdm_cq.h b/prov/efa/src/rdm/efa_rdm_cq.h index a56d62dac40..e1a865ee127 100644 --- a/prov/efa/src/rdm/efa_rdm_cq.h +++ b/prov/efa/src/rdm/efa_rdm_cq.h @@ -8,8 +8,7 @@ #include struct efa_rdm_cq { - struct util_cq util_cq; - struct efa_ibv_cq ibv_cq; + struct efa_cq efa_cq; struct fid_cq *shm_cq; struct dlist_entry ibv_cq_poll_list; bool need_to_scan_ep_list; diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c index 3508fe7ba75..1981ed9825f 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_fiops.c +++ b/prov/efa/src/rdm/efa_rdm_ep_fiops.c @@ -37,13 +37,13 @@ void efa_rdm_ep_construct_ibv_qp_init_attr_ex(struct efa_rdm_ep *ep, static inline struct efa_rdm_cq *efa_rdm_ep_get_tx_rdm_cq(struct efa_rdm_ep *ep) { - return ep->base_ep.util_ep.tx_cq ? container_of(ep->base_ep.util_ep.tx_cq, struct efa_rdm_cq, util_cq) : NULL; + return ep->base_ep.util_ep.tx_cq ? container_of(ep->base_ep.util_ep.tx_cq, struct efa_rdm_cq, efa_cq.util_cq) : NULL; } static inline struct efa_rdm_cq *efa_rdm_ep_get_rx_rdm_cq(struct efa_rdm_ep *ep) { - return ep->base_ep.util_ep.rx_cq ? container_of(ep->base_ep.util_ep.rx_cq, struct efa_rdm_cq, util_cq) : NULL; + return ep->base_ep.util_ep.rx_cq ? container_of(ep->base_ep.util_ep.rx_cq, struct efa_rdm_cq, efa_cq.util_cq) : NULL; } /** @@ -58,33 +58,33 @@ static int efa_rdm_ep_create_base_ep_ibv_qp(struct efa_rdm_ep *ep) { struct ibv_qp_init_attr_ex attr_ex = { 0 }; - struct efa_rdm_cq *tx_rdm_cq, *rx_rdm_cq; + struct efa_cq *tx_cq, *rx_cq; struct ibv_cq_ex *tx_ibv_cq, *rx_ibv_cq; int ret; - tx_rdm_cq = efa_rdm_ep_get_tx_rdm_cq(ep); - rx_rdm_cq = efa_rdm_ep_get_rx_rdm_cq(ep); + tx_cq = efa_base_ep_get_tx_cq(&ep->base_ep); + rx_cq = efa_base_ep_get_rx_cq(&ep->base_ep); - if (!tx_rdm_cq && !rx_rdm_cq) { + if (!tx_cq && !rx_cq) { EFA_WARN(FI_LOG_EP_CTRL, "Endpoint is not bound to a send or receive completion queue\n"); return -FI_ENOCQ; } - if (!tx_rdm_cq && ofi_needs_tx(ep->base_ep.info->caps)) { + if (!tx_cq && ofi_needs_tx(ep->base_ep.info->caps)) { EFA_WARN(FI_LOG_EP_CTRL, "Endpoint is not bound to a send completion queue when it has transmit capabilities enabled (FI_SEND).\n"); return -FI_ENOCQ; } - if (!rx_rdm_cq && ofi_needs_rx(ep->base_ep.info->caps)) { + if (!rx_cq && ofi_needs_rx(ep->base_ep.info->caps)) { EFA_WARN(FI_LOG_EP_CTRL, "Endpoint is not bound to a receive completion queue when it has receive capabilities enabled (FI_RECV).\n"); return -FI_ENOCQ; } - tx_ibv_cq = tx_rdm_cq ? tx_rdm_cq->ibv_cq.ibv_cq_ex : rx_rdm_cq->ibv_cq.ibv_cq_ex; - rx_ibv_cq = rx_rdm_cq ? rx_rdm_cq->ibv_cq.ibv_cq_ex : tx_rdm_cq->ibv_cq.ibv_cq_ex; + tx_ibv_cq = tx_cq ? tx_cq->ibv_cq.ibv_cq_ex : rx_cq->ibv_cq.ibv_cq_ex; + rx_ibv_cq = rx_cq ? rx_cq->ibv_cq.ibv_cq_ex : tx_cq->ibv_cq.ibv_cq_ex; efa_rdm_ep_construct_ibv_qp_init_attr_ex(ep, &attr_ex, tx_ibv_cq, rx_ibv_cq); @@ -699,9 +699,9 @@ static int efa_rdm_ep_bind(struct fid *ep_fid, struct fid *bfid, uint64_t flags) } break; case FI_CLASS_CQ: - cq = container_of(bfid, struct efa_rdm_cq, util_cq.cq_fid.fid); + cq = container_of(bfid, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); - ret = ofi_ep_bind_cq(&efa_rdm_ep->base_ep.util_ep, &cq->util_cq, flags); + ret = ofi_ep_bind_cq(&efa_rdm_ep->base_ep.util_ep, &cq->efa_cq.util_cq, flags); if (ret) return ret; @@ -873,12 +873,12 @@ bool efa_rdm_ep_has_unfinished_send(struct efa_rdm_ep *efa_rdm_ep) static inline void efa_rdm_ep_wait_send(struct efa_rdm_ep *efa_rdm_ep) { - struct efa_rdm_cq *tx_cq, *rx_cq; + struct efa_cq *tx_cq, *rx_cq; ofi_genlock_lock(&efa_rdm_ep_domain(efa_rdm_ep)->srx_lock); - tx_cq = efa_rdm_ep_get_tx_rdm_cq(efa_rdm_ep); - rx_cq = efa_rdm_ep_get_rx_rdm_cq(efa_rdm_ep); + tx_cq = efa_base_ep_get_tx_cq(&efa_rdm_ep->base_ep); + rx_cq = efa_base_ep_get_rx_cq(&efa_rdm_ep->base_ep); while (efa_rdm_ep_has_unfinished_send(efa_rdm_ep)) { /* poll cq until empty */ @@ -898,10 +898,10 @@ void efa_rdm_ep_remove_cntr_ibv_cq_poll_list(struct efa_rdm_ep *ep) int i; struct efa_cntr *efa_cntr; struct util_cntr *util_cntr; - struct efa_rdm_cq *tx_cq, *rx_cq; + struct efa_cq *tx_cq, *rx_cq; - tx_cq = efa_rdm_ep_get_tx_rdm_cq(ep); - rx_cq = efa_rdm_ep_get_rx_rdm_cq(ep); + tx_cq = efa_base_ep_get_tx_cq(&ep->base_ep); + rx_cq = efa_base_ep_get_rx_cq(&ep->base_ep); for (i = 0; i< CNTR_CNT; i++) { util_cntr = ep->base_ep.util_ep.cntrs[i]; @@ -928,16 +928,16 @@ void efa_rdm_ep_remove_cq_ibv_cq_poll_list(struct efa_rdm_ep *ep) * It must happen after ofi_endpoint_close * so we have cq's reference counters updated. */ - if (tx_cq && !ofi_atomic_get32(&tx_cq->util_cq.ref)) { - efa_ibv_cq_poll_list_remove(&tx_cq->ibv_cq_poll_list, &tx_cq->util_cq.ep_list_lock, &tx_cq->ibv_cq); + if (tx_cq && !ofi_atomic_get32(&tx_cq->efa_cq.util_cq.ref)) { + efa_ibv_cq_poll_list_remove(&tx_cq->ibv_cq_poll_list, &tx_cq->efa_cq.util_cq.ep_list_lock, &tx_cq->efa_cq.ibv_cq); if (rx_cq) - efa_ibv_cq_poll_list_remove(&rx_cq->ibv_cq_poll_list, &rx_cq->util_cq.ep_list_lock, &tx_cq->ibv_cq); + efa_ibv_cq_poll_list_remove(&rx_cq->ibv_cq_poll_list, &rx_cq->efa_cq.util_cq.ep_list_lock, &tx_cq->efa_cq.ibv_cq); } - if (rx_cq && !ofi_atomic_get32(&rx_cq->util_cq.ref)) { - efa_ibv_cq_poll_list_remove(&rx_cq->ibv_cq_poll_list, &rx_cq->util_cq.ep_list_lock, &rx_cq->ibv_cq); + if (rx_cq && !ofi_atomic_get32(&rx_cq->efa_cq.util_cq.ref)) { + efa_ibv_cq_poll_list_remove(&rx_cq->ibv_cq_poll_list, &rx_cq->efa_cq.util_cq.ep_list_lock, &rx_cq->efa_cq.ibv_cq); if (tx_cq) - efa_ibv_cq_poll_list_remove(&tx_cq->ibv_cq_poll_list, &tx_cq->util_cq.ep_list_lock, &rx_cq->ibv_cq); + efa_ibv_cq_poll_list_remove(&tx_cq->ibv_cq_poll_list, &tx_cq->efa_cq.util_cq.ep_list_lock, &rx_cq->efa_cq.ibv_cq); } } @@ -1099,7 +1099,7 @@ static void efa_rdm_ep_close_shm_resources(struct efa_rdm_ep *efa_rdm_ep) efa_av->shm_rdm_av = NULL; } - efa_rdm_cq = container_of(efa_rdm_ep->base_ep.util_ep.tx_cq, struct efa_rdm_cq, util_cq); + efa_rdm_cq = container_of(efa_rdm_ep->base_ep.util_ep.tx_cq, struct efa_rdm_cq, efa_cq.util_cq); if (efa_rdm_cq->shm_cq) { ret = fi_close(&efa_rdm_cq->shm_cq->fid); if (ret) @@ -1107,7 +1107,7 @@ static void efa_rdm_ep_close_shm_resources(struct efa_rdm_ep *efa_rdm_ep) efa_rdm_cq->shm_cq = NULL; } - efa_rdm_cq = container_of(efa_rdm_ep->base_ep.util_ep.rx_cq, struct efa_rdm_cq, util_cq); + efa_rdm_cq = container_of(efa_rdm_ep->base_ep.util_ep.rx_cq, struct efa_rdm_cq, efa_cq.util_cq); if (efa_rdm_cq->shm_cq) { ret = fi_close(&efa_rdm_cq->shm_cq->fid); if (ret) @@ -1187,9 +1187,9 @@ int efa_rdm_ep_insert_cntr_ibv_cq_poll_list(struct efa_rdm_ep *ep) int i, ret; struct efa_cntr *efa_cntr; struct util_cntr *util_cntr; - struct efa_rdm_cq *tx_cq, *rx_cq; - tx_cq = efa_rdm_ep_get_tx_rdm_cq(ep); - rx_cq = efa_rdm_ep_get_rx_rdm_cq(ep); + struct efa_cq *tx_cq, *rx_cq; + tx_cq = efa_base_ep_get_tx_cq(&ep->base_ep); + rx_cq = efa_base_ep_get_rx_cq(&ep->base_ep); for (i = 0; i < CNTR_CNT; i++) { util_cntr = ep->base_ep.util_ep.cntrs[i]; @@ -1224,33 +1224,33 @@ int efa_rdm_ep_insert_cq_ibv_cq_poll_list(struct efa_rdm_ep *ep) rx_cq = efa_rdm_ep_get_rx_rdm_cq(ep); if (tx_cq) { - ret = efa_ibv_cq_poll_list_insert(&tx_cq->ibv_cq_poll_list, &tx_cq->util_cq.ep_list_lock, &tx_cq->ibv_cq); + ret = efa_ibv_cq_poll_list_insert(&tx_cq->ibv_cq_poll_list, &tx_cq->efa_cq.util_cq.ep_list_lock, &tx_cq->efa_cq.ibv_cq); if (ret) return ret; if (rx_cq) { - ret = efa_ibv_cq_poll_list_insert(&tx_cq->ibv_cq_poll_list, &tx_cq->util_cq.ep_list_lock, &rx_cq->ibv_cq); + ret = efa_ibv_cq_poll_list_insert(&tx_cq->ibv_cq_poll_list, &tx_cq->efa_cq.util_cq.ep_list_lock, &rx_cq->efa_cq.ibv_cq); if (ret) return ret; } - ofi_genlock_lock(&tx_cq->util_cq.ep_list_lock); + ofi_genlock_lock(&tx_cq->efa_cq.util_cq.ep_list_lock); tx_cq->need_to_scan_ep_list = true; - ofi_genlock_unlock(&tx_cq->util_cq.ep_list_lock); + ofi_genlock_unlock(&tx_cq->efa_cq.util_cq.ep_list_lock); } if (rx_cq) { - ret = efa_ibv_cq_poll_list_insert(&rx_cq->ibv_cq_poll_list, &rx_cq->util_cq.ep_list_lock, &rx_cq->ibv_cq); + ret = efa_ibv_cq_poll_list_insert(&rx_cq->ibv_cq_poll_list, &rx_cq->efa_cq.util_cq.ep_list_lock, &rx_cq->efa_cq.ibv_cq); if (ret) return ret; if (tx_cq) { - ret = efa_ibv_cq_poll_list_insert(&rx_cq->ibv_cq_poll_list, &rx_cq->util_cq.ep_list_lock, &tx_cq->ibv_cq); + ret = efa_ibv_cq_poll_list_insert(&rx_cq->ibv_cq_poll_list, &rx_cq->efa_cq.util_cq.ep_list_lock, &tx_cq->efa_cq.ibv_cq); if (ret) return ret; } - ofi_genlock_lock(&rx_cq->util_cq.ep_list_lock); + ofi_genlock_lock(&rx_cq->efa_cq.util_cq.ep_list_lock); rx_cq->need_to_scan_ep_list = true; - ofi_genlock_unlock(&rx_cq->util_cq.ep_list_lock); + ofi_genlock_unlock(&rx_cq->efa_cq.util_cq.ep_list_lock); } return FI_SUCCESS; diff --git a/prov/efa/test/efa_unit_test_cq.c b/prov/efa/test/efa_unit_test_cq.c index c5b93cd5e66..e69fb8b432e 100644 --- a/prov/efa/test/efa_unit_test_cq.c +++ b/prov/efa/test/efa_unit_test_cq.c @@ -20,22 +20,13 @@ void test_impl_cq_read_empty_cq(struct efa_resource *resource, enum fi_ep_type e struct ibv_cq_ex *ibv_cqx; struct fi_cq_data_entry cq_entry; int ret; + struct efa_base_ep *efa_base_ep; efa_unit_test_resource_construct(resource, ep_type); - if (ep_type == FI_EP_DGRAM) { - struct efa_dgram_ep *efa_dgram_ep; - - efa_dgram_ep = container_of(resource->ep, struct efa_dgram_ep, base_ep.util_ep.ep_fid); - ibv_cqx = container_of(efa_dgram_ep->base_ep.util_ep.rx_cq, struct efa_cq, util_cq)->ibv_cq.ibv_cq_ex; - } else { - struct efa_rdm_ep *efa_rdm_ep; - - efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); - assert(efa_rdm_ep->base_ep.util_ep.rx_cq); - ibv_cqx = container_of(efa_rdm_ep->base_ep.util_ep.rx_cq, struct efa_rdm_cq, util_cq)->ibv_cq.ibv_cq_ex; - } + efa_base_ep = container_of(resource->ep, struct efa_base_ep, util_ep.ep_fid); + ibv_cqx = container_of(efa_base_ep->util_ep.rx_cq, struct efa_cq, util_cq)->ibv_cq.ibv_cq_ex; ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock; /* ibv_start_poll to return ENOENT means device CQ is empty */ @@ -110,8 +101,8 @@ static void test_rdm_cq_read_bad_send_status(struct efa_resource *resource, efa_rdm_ep->host_id = local_host_id; ibv_qpx = efa_rdm_ep->base_ep.qp->ibv_qp_ex; - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); - ibv_cqx = efa_rdm_cq->ibv_cq.ibv_cq_ex; + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); + ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex; ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len); assert_int_equal(ret, 0); @@ -296,6 +287,7 @@ void test_ibv_cq_ex_read_bad_recv_status(struct efa_resource **state) struct fi_eq_err_entry eq_err_entry; int ret; struct efa_rdm_cq *efa_rdm_cq; + struct ibv_cq_ex *ibv_cqx; efa_unit_test_resource_construct(resource, FI_EP_RDM); @@ -314,13 +306,14 @@ void test_ibv_cq_ex_read_bad_recv_status(struct efa_resource **state) assert_non_null(pkt_entry); efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep); - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); + ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex; - efa_rdm_cq->ibv_cq.ibv_cq_ex->start_poll = &efa_mock_ibv_start_poll_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->end_poll = &efa_mock_ibv_end_poll_check_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_opcode = &efa_mock_ibv_read_opcode_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; + ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock; + ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock; + ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock; + ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; + ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; will_return(efa_mock_ibv_start_poll_return_mock, 0); will_return(efa_mock_ibv_end_poll_check_mock, NULL); @@ -334,12 +327,12 @@ void test_ibv_cq_ex_read_bad_recv_status(struct efa_resource **state) /* the recv error will not populate to application cq because it's an EFA internal error and * and not related to any application recv. Currently we can only read the error from eq. */ - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = (uintptr_t)pkt_entry; - efa_rdm_cq->ibv_cq.ibv_cq_ex->status = IBV_WC_GENERAL_ERR; + ibv_cqx->wr_id = (uintptr_t)pkt_entry; + ibv_cqx->status = IBV_WC_GENERAL_ERR; #if HAVE_CAPS_UNSOLICITED_WRITE_RECV if (efa_use_unsolicited_write_recv()) { - efadv_cq_from_ibv_cq_ex(efa_rdm_cq->ibv_cq.ibv_cq_ex)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited; + efadv_cq_from_ibv_cq_ex(ibv_cqx)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited; will_return(efa_mock_efadv_wc_is_unsolicited, false); } #endif @@ -372,18 +365,20 @@ void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource struct fi_eq_err_entry eq_err_entry; int ret; struct efa_rdm_cq *efa_rdm_cq; + struct ibv_cq_ex *ibv_cqx; efa_unit_test_resource_construct(resource, FI_EP_RDM); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); + ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex; - efa_rdm_cq->ibv_cq.ibv_cq_ex->start_poll = &efa_mock_ibv_start_poll_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->end_poll = &efa_mock_ibv_end_poll_check_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_opcode = &efa_mock_ibv_read_opcode_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; + ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock; + ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock; + ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock; + ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; + ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; will_return(efa_mock_ibv_start_poll_return_mock, 0); will_return(efa_mock_ibv_end_poll_check_mock, NULL); @@ -399,10 +394,10 @@ void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource #if HAVE_CAPS_UNSOLICITED_WRITE_RECV if (use_unsolicited_recv) { - efadv_cq_from_ibv_cq_ex(efa_rdm_cq->ibv_cq.ibv_cq_ex)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited; + efadv_cq_from_ibv_cq_ex(ibv_cqx)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited; will_return(efa_mock_efa_device_support_unsolicited_write_recv, true); will_return(efa_mock_efadv_wc_is_unsolicited, true); - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = 0; + ibv_cqx->wr_id = 0; } else { /* * For solicited write recv, it will consume an internal rx pkt @@ -411,7 +406,7 @@ void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource struct efa_rdm_pke *pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL); assert_non_null(pkt_entry); efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep); - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = (uintptr_t)pkt_entry; + ibv_cqx->wr_id = (uintptr_t)pkt_entry; } #else /* @@ -421,12 +416,12 @@ void test_ibv_cq_ex_read_bad_recv_rdma_with_imm_status_impl(struct efa_resource struct efa_rdm_pke *pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, EFA_RDM_PKE_FROM_EFA_RX_POOL); assert_non_null(pkt_entry); efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep); - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = (uintptr_t)pkt_entry; + ibv_cqx->wr_id = (uintptr_t)pkt_entry; #endif /* the recv rdma with imm will not populate to application cq because it's an EFA internal error and * and not related to any application operations. Currently we can only read the error from eq. */ - efa_rdm_cq->ibv_cq.ibv_cq_ex->status = IBV_WC_GENERAL_ERR; + ibv_cqx->status = IBV_WC_GENERAL_ERR; ret = fi_cq_read(resource->cq, &cq_entry, 1); assert_int_equal(ret, -FI_EAGAIN); @@ -460,13 +455,16 @@ void test_ibv_cq_ex_read_failed_poll(struct efa_resource **state) struct fi_cq_err_entry cq_err_entry; int ret; struct efa_rdm_cq *efa_rdm_cq; + struct ibv_cq_ex *ibv_cqx; efa_unit_test_resource_construct(resource, FI_EP_RDM); - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); - efa_rdm_cq->ibv_cq.ibv_cq_ex->start_poll = &efa_mock_ibv_start_poll_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->end_poll = &efa_mock_ibv_end_poll_check_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); + ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex; + + ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock; + ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock; + ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; will_return(efa_mock_ibv_start_poll_return_mock, EFAULT); will_return(efa_mock_ibv_read_vendor_err_return_mock, EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE); @@ -536,7 +534,7 @@ int test_efa_rdm_cq_get_ibv_cq_poll_list_length(struct fid_cq *cq_fid) { struct efa_rdm_cq *cq; - cq = container_of(cq_fid, struct efa_rdm_cq, util_cq.cq_fid.fid); + cq = container_of(cq_fid, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); return efa_unit_test_get_dlist_length(&cq->ibv_cq_poll_list); } @@ -598,7 +596,7 @@ void test_efa_rdm_cq_post_initial_rx_pkts(struct efa_resource **state) efa_unit_test_resource_construct(resource, FI_EP_RDM); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); /* At this time, rx pkts are not growed and posted */ assert_int_equal(efa_rdm_ep->efa_rx_pkts_to_post, 0); @@ -641,6 +639,7 @@ static void test_impl_ibv_cq_ex_read_unknow_peer_ah(struct efa_resource *resourc struct efa_unit_test_buff recv_buff; int ret; struct efa_rdm_cq *efa_rdm_cq; + struct ibv_cq_ex *ibv_cqx; /* * Always use mocked efadv_create_cq instead of the real one. @@ -659,7 +658,8 @@ static void test_impl_ibv_cq_ex_read_unknow_peer_ah(struct efa_resource *resourc efa_unit_test_resource_construct(resource, FI_EP_RDM); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); + ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex; /* Construct a minimal recv buffer */ efa_unit_test_buff_construct(&recv_buff, resource, efa_rdm_ep->min_multi_recv_size); @@ -698,19 +698,19 @@ static void test_impl_ibv_cq_ex_read_unknow_peer_ah(struct efa_resource *resourc efa_unit_test_eager_msgrtm_pkt_construct(pkt_entry, &pkt_attr); /* Setup CQ */ - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = (uintptr_t)pkt_entry; - efa_rdm_cq->ibv_cq.ibv_cq_ex->start_poll = &efa_mock_ibv_start_poll_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->next_poll = &efa_mock_ibv_next_poll_check_function_called_and_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->end_poll = &efa_mock_ibv_end_poll_check_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_slid = &efa_mock_ibv_read_slid_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_opcode = &efa_mock_ibv_read_opcode_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock; + ibv_cqx->wr_id = (uintptr_t)pkt_entry; + ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock; + ibv_cqx->next_poll = &efa_mock_ibv_next_poll_check_function_called_and_return_mock; + ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock; + ibv_cqx->read_slid = &efa_mock_ibv_read_slid_return_mock; + ibv_cqx->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock; + ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock; + ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; + ibv_cqx->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock; + ibv_cqx->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock; if (support_efadv_cq) { - efadv_cq = efadv_cq_from_ibv_cq_ex(efa_rdm_cq->ibv_cq.ibv_cq_ex); + efadv_cq = efadv_cq_from_ibv_cq_ex(ibv_cqx); assert_non_null(efadv_cq); efadv_cq->wc_read_sgid = &efa_mock_efadv_wc_read_sgid_return_zero_code_and_expect_next_poll_and_set_gid; diff --git a/prov/efa/test/efa_unit_test_ep.c b/prov/efa/test/efa_unit_test_ep.c index d64139b986c..f2d1d1f0e7a 100644 --- a/prov/efa/test/efa_unit_test_ep.c +++ b/prov/efa/test/efa_unit_test_ep.c @@ -110,6 +110,7 @@ void test_efa_rdm_ep_handshake_exchange_host_id(struct efa_resource **state, uin struct efa_rdm_pke *pkt_entry; uint64_t actual_peer_host_id = UINT64_MAX; struct efa_rdm_cq *efa_rdm_cq; + struct ibv_cq_ex *ibv_cqx; g_efa_unit_test_mocks.local_host_id = local_host_id; g_efa_unit_test_mocks.peer_host_id = peer_host_id; @@ -120,7 +121,8 @@ void test_efa_rdm_ep_handshake_exchange_host_id(struct efa_resource **state, uin efa_unit_test_resource_construct_rdm_shm_disabled(resource); efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid); - efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, util_cq.cq_fid.fid); + efa_rdm_cq = container_of(resource->cq, struct efa_rdm_cq, efa_cq.util_cq.cq_fid.fid); + ibv_cqx = efa_rdm_cq->efa_cq.ibv_cq.ibv_cq_ex; efa_rdm_ep->host_id = g_efa_unit_test_mocks.local_host_id; @@ -166,18 +168,18 @@ void test_efa_rdm_ep_handshake_exchange_host_id(struct efa_resource **state, uin expect_function_call(efa_mock_ibv_wr_send_verify_handshake_pkt_local_host_id_and_save_wr); /* Setup CQ */ - efa_rdm_cq->ibv_cq.ibv_cq_ex->end_poll = &efa_mock_ibv_end_poll_check_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->next_poll = &efa_mock_ibv_next_poll_check_function_called_and_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_opcode = &efa_mock_ibv_read_opcode_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_slid = &efa_mock_ibv_read_slid_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->start_poll = &efa_mock_ibv_start_poll_return_mock; - efa_rdm_cq->ibv_cq.ibv_cq_ex->status = IBV_WC_SUCCESS; - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = (uintptr_t)pkt_entry; + ibv_cqx->end_poll = &efa_mock_ibv_end_poll_check_mock; + ibv_cqx->next_poll = &efa_mock_ibv_next_poll_check_function_called_and_return_mock; + ibv_cqx->read_byte_len = &efa_mock_ibv_read_byte_len_return_mock; + ibv_cqx->read_opcode = &efa_mock_ibv_read_opcode_return_mock; + ibv_cqx->read_slid = &efa_mock_ibv_read_slid_return_mock; + ibv_cqx->read_src_qp = &efa_mock_ibv_read_src_qp_return_mock; + ibv_cqx->read_qp_num = &efa_mock_ibv_read_qp_num_return_mock; + ibv_cqx->read_wc_flags = &efa_mock_ibv_read_wc_flags_return_mock; + ibv_cqx->read_vendor_err = &efa_mock_ibv_read_vendor_err_return_mock; + ibv_cqx->start_poll = &efa_mock_ibv_start_poll_return_mock; + ibv_cqx->status = IBV_WC_SUCCESS; + ibv_cqx->wr_id = (uintptr_t)pkt_entry; expect_function_call(efa_mock_ibv_next_poll_check_function_called_and_return_mock); /* Receive handshake packet */ @@ -210,8 +212,8 @@ void test_efa_rdm_ep_handshake_exchange_host_id(struct efa_resource **state, uin * We need to poll the CQ twice explicitly to point the CQE * to the saved send wr in handshake */ - efa_rdm_cq->ibv_cq.ibv_cq_ex->status = IBV_WC_GENERAL_ERR; - efa_rdm_cq->ibv_cq.ibv_cq_ex->wr_id = (uintptr_t)g_ibv_submitted_wr_id_vec[0]; + ibv_cqx->status = IBV_WC_GENERAL_ERR; + ibv_cqx->wr_id = (uintptr_t)g_ibv_submitted_wr_id_vec[0]; /* Progress the send wr to clean up outstanding tx ops */ cq_read_send_ret = fi_cq_read(resource->cq, &cq_entry, 1);