From c3bc905ecaf5c08e4ea9bcae14c3d08008675c39 Mon Sep 17 00:00:00 2001 From: Sylvain Didelot Date: Wed, 27 Sep 2023 16:21:51 +0200 Subject: [PATCH] prov/verbs: Add support for IBV_ACCESS_RELAXED_ORDERING IBV_ACCESS_RELAXED_ORDERING allows the system to reorder Send/Write/Atomic operations to improve performance. The patch enables IBV_ACCESS_RELAXED_ORDERING if the application has requested no ordering in TX/RX attributes. Signed-off-by: Sylvain Didelot --- prov/verbs/configure.m4 | 10 ++++++++++ prov/verbs/src/verbs_mr.c | 5 +++++ prov/verbs/src/verbs_ofi.h | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/prov/verbs/configure.m4 b/prov/verbs/configure.m4 index aa793e0180b..b096e9cae82 100644 --- a/prov/verbs/configure.m4 +++ b/prov/verbs/configure.m4 @@ -93,6 +93,16 @@ AC_DEFUN([FI_VERBS_CONFIGURE],[ AC_DEFINE_UNQUOTED([VERBS_HAVE_DMABUF_MR],[$VERBS_HAVE_DMABUF_MR], [Whether infiniband/verbs.h has ibv_reg_dmabuf_mr() support or not]) + #See if we have rdma-core IBV_ACCESS_RELAXED_ORDERING mr support + VERBS_HAVE_RELAXED_ORDERING_MR=0 + AS_IF([test $verbs_ibverbs_happy -eq 1],[ + AC_CHECK_DECL([IBV_ACCESS_RELAXED_ORDERING], + [VERBS_HAVE_RELAXED_ORDERING_MR=1],[], + [#include ]) + ]) + AC_DEFINE_UNQUOTED([VERBS_HAVE_RELAXED_ORDERING_MR],[$VERBS_HAVE_RELAXED_ORDERING_MR], + [Whether infiniband/verbs.h has IBV_ACCESS_RELAXED_ORDERING support or not]) + CPPFLAGS=$fi_verbs_configure_save_CPPFLAGS # Technically, verbs_ibverbs_CPPFLAGS and diff --git a/prov/verbs/src/verbs_mr.c b/prov/verbs/src/verbs_mr.c index 360094432a2..88d998f7e36 100644 --- a/prov/verbs/src/verbs_mr.c +++ b/prov/verbs/src/verbs_mr.c @@ -201,6 +201,11 @@ vrb_mr_ofi2ibv_access(uint64_t ofi_access, struct vrb_domain *domain) IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_ATOMIC; + /* No strict ordering requested */ + if (domain->util_domain.tx_msg_order == FI_ORDER_NONE && + domain->util_domain.rx_msg_order == FI_ORDER_NONE) + ibv_access |= VRB_ACCESS_RELAXED_ORDERING; + return ibv_access; } diff --git a/prov/verbs/src/verbs_ofi.h b/prov/verbs/src/verbs_ofi.h index 78b8c40369e..7f340acbf1d 100644 --- a/prov/verbs/src/verbs_ofi.h +++ b/prov/verbs/src/verbs_ofi.h @@ -437,6 +437,12 @@ int vrb_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, struct fid_cq **cq, void *context); int vrb_cq_trywait(struct vrb_cq *cq); +#if VERBS_HAVE_RELAXED_ORDERING_MR +#define VRB_ACCESS_RELAXED_ORDERING IBV_ACCESS_RELAXED_ORDERING +#else +#define VRB_ACCESS_RELAXED_ORDERING 0 +#endif + struct vrb_mem_desc { struct fid_mr mr_fid; struct ibv_mr *mr;