Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

prov/efa: Add new efa-direct fi_info objects #10735

Merged
merged 3 commits into from
Feb 7, 2025
Merged

prov/efa: Add new efa-direct fi_info objects #10735

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
14e6789
prov/efa: Add new efa-direct fi_info objects
sunkuamzn Jan 29, 2025
04b0f78
prov/efa: Remove incorrect usage of rdm_info->ep_attr->max_msg_size
sunkuamzn Feb 5, 2025
ea02120
prov/efa: Unit test additions and fixes for efa-direct
sunkuamzn Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions prov/efa/test/efa_unit_test_av.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,39 @@
/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */

#include "efa_unit_tests.h"
#include "efa_av.h"

/**
* @brief Verify the ep type in struct efa_av for efa RDM path
*
* @param[in] state struct efa_resource that is managed by the framework
*/
void test_av_ep_type_efa_rdm(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_av *efa_av;

efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
g_efa_unit_test_mocks.ibv_create_ah = &efa_mock_ibv_create_ah_check_mock;
efa_av = container_of(resource->av, struct efa_av, util_av.av_fid);
assert(efa_av->ep_type == FI_EP_RDM);
}

/**
* @brief Verify the ep type in struct efa_av for efa direct path
*
* @param[in] state struct efa_resource that is managed by the framework
*/
void test_av_ep_type_efa_direct(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_av *efa_av;

efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
g_efa_unit_test_mocks.ibv_create_ah = &efa_mock_ibv_create_ah_check_mock;
efa_av = container_of(resource->av, struct efa_av, util_av.av_fid);
assert(efa_av->ep_type == FI_EP_RDM);
}

/**
* @brief Only works on nodes with EFA devices
Expand Down
62 changes: 26 additions & 36 deletions prov/efa/test/efa_unit_test_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,37 +52,41 @@ void efa_unit_test_construct_msg_rma(struct fi_msg_rma *msg, struct iovec *iov,
msg->data = data;
}

struct fi_info *efa_unit_test_alloc_hints(enum fi_ep_type ep_type, char *prov_name)
struct fi_info *efa_unit_test_alloc_hints(enum fi_ep_type ep_type, char *fabric_name)
{
struct fi_info *hints;

hints = fi_allocinfo();
if (!hints)
return NULL;

hints->fabric_attr->prov_name = strdup(prov_name);
if (fabric_name)
hints->fabric_attr->name = strdup(fabric_name);
hints->ep_attr->type = ep_type;

/* Use a minimal caps that efa / efa-direct should always support */
hints->domain_attr->mr_mode = MR_MODE_BITS;

/* EFA direct and dgram paths require FI_CONTEXT2 */
if (!fabric_name || !strcasecmp(fabric_name, EFA_DIRECT_FABRIC_NAME))
hints->mode |= FI_CONTEXT2;

if (ep_type == FI_EP_DGRAM) {
hints->mode |= FI_MSG_PREFIX;
hints->mode |= FI_MSG_PREFIX | FI_CONTEXT2;
}

return hints;
}

/* TODO: remove use_efa_direct after we have efa_direct implemented in fi_info */
void efa_unit_test_resource_construct_with_hints(struct efa_resource *resource,
enum fi_ep_type ep_type,
uint32_t fi_version, struct fi_info *hints,
bool enable_ep, bool open_cq, char* prov_name)
bool enable_ep, bool open_cq)
{
int ret = 0;
struct fi_av_attr av_attr = {0};
struct fi_cq_attr cq_attr = {0};
struct fi_eq_attr eq_attr = {0};
struct efa_domain *efa_domain;

ret = fi_getinfo(fi_version, NULL, NULL, 0ULL, hints, &resource->info);
if (ret)
Expand All @@ -96,17 +100,6 @@ void efa_unit_test_resource_construct_with_hints(struct efa_resource *resource,
if (ret)
goto err;

/*
* TODO: Remove this function pointer override when we have it assigned
* for efa-direct correctly.
*/
if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name)) {
efa_domain = container_of(resource->domain, struct efa_domain, util_domain.domain_fid);

efa_domain->util_domain.domain_fid.ops->endpoint = efa_ep_open;
efa_domain->util_domain.domain_fid.ops->cq_open = efa_cq_open;
}

ret = fi_endpoint(resource->domain, resource->info, &resource->ep, NULL);
if (ret)
goto err;
Expand Down Expand Up @@ -146,19 +139,18 @@ void efa_unit_test_resource_construct_with_hints(struct efa_resource *resource,
assert_int_equal(ret, 0);
}

void efa_unit_test_resource_construct(struct efa_resource *resource, enum fi_ep_type ep_type, char *prov_name)
void efa_unit_test_resource_construct(struct efa_resource *resource, enum fi_ep_type ep_type, char *fabric_name)
{

/* TODO use prov_name here when efa-direct fi_info is implemented */
resource->hints = efa_unit_test_alloc_hints(ep_type, EFA_PROV_NAME);
resource->hints = efa_unit_test_alloc_hints(ep_type, fabric_name);
if (!resource->hints)
goto err;
if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name))
if (!strcmp(EFA_DIRECT_FABRIC_NAME, fabric_name))
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(2, 0),
resource->hints, true, true, prov_name);
resource->hints, true, true);
else
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(1, 14),
resource->hints, true, true, prov_name);
resource->hints, true, true);
return;

err:
Expand All @@ -169,19 +161,18 @@ void efa_unit_test_resource_construct(struct efa_resource *resource, enum fi_ep_
}

void efa_unit_test_resource_construct_ep_not_enabled(struct efa_resource *resource,
enum fi_ep_type ep_type, char *prov_name)
enum fi_ep_type ep_type, char *fabric_name)
{
/* TODO use prov_name here when efa-direct fi_info is implemented */
resource->hints = efa_unit_test_alloc_hints(ep_type, EFA_PROV_NAME);
resource->hints = efa_unit_test_alloc_hints(ep_type, fabric_name);
if (!resource->hints)
goto err;

if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name))
if (!strcmp(EFA_DIRECT_FABRIC_NAME, fabric_name))
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(2, 0),
resource->hints, false, true, prov_name);
resource->hints, false, true);
else
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(1, 14),
resource->hints, false, true, prov_name);
resource->hints, false, true);
return;

err:
Expand All @@ -192,19 +183,18 @@ void efa_unit_test_resource_construct_ep_not_enabled(struct efa_resource *resour
}

void efa_unit_test_resource_construct_no_cq_and_ep_not_enabled(struct efa_resource *resource,
enum fi_ep_type ep_type, char *prov_name)
enum fi_ep_type ep_type, char *fabric_name)
{
/* TODO use prov_name here when efa-direct fi_info is implemented */
resource->hints = efa_unit_test_alloc_hints(ep_type, EFA_PROV_NAME);
resource->hints = efa_unit_test_alloc_hints(ep_type, fabric_name);
if (!resource->hints)
goto err;

if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name))
if (!strcmp(EFA_DIRECT_FABRIC_NAME, fabric_name))
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(2, 0),
resource->hints, false, false, prov_name);
resource->hints, false, false);
else
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(1, 14),
resource->hints, false, false, prov_name);
resource->hints, false, false);
return;

err:
Expand All @@ -227,7 +217,7 @@ void efa_unit_test_resource_construct_rdm_shm_disabled(struct efa_resource *reso
goto err;

efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, false, true, EFA_PROV_NAME);
resource->hints, false, true);

ret = fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT,
FI_OPT_SHARED_MEMORY_PERMITTED, &shm_permitted,
Expand Down
12 changes: 6 additions & 6 deletions prov/efa/test/efa_unit_test_ep.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@ void test_efa_rdm_ep_rma_queue_before_handshake(struct efa_resource **state, int
resource->hints->caps |= FI_MSG | FI_TAGGED | FI_RMA;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

/* ensure we don't have RMA capability. */
efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
Expand Down Expand Up @@ -685,7 +685,7 @@ void test_efa_rdm_ep_rma_inconsistent_unsolicited_write_recv(struct efa_resource
resource->hints->caps |= FI_MSG | FI_TAGGED | FI_RMA;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 22),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);

Expand Down Expand Up @@ -794,7 +794,7 @@ void test_efa_rdm_ep_rma_without_caps(struct efa_resource **state)
resource->hints->caps &= ~FI_RMA;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

/* ensure we don't have RMA capability. */
efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
Expand Down Expand Up @@ -845,7 +845,7 @@ void test_efa_rdm_ep_atomic_without_caps(struct efa_resource **state)
resource->hints->caps &= ~FI_ATOMIC;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

/* ensure we don't have ATOMIC capability. */
efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
Expand Down Expand Up @@ -1001,7 +1001,7 @@ static void test_efa_rdm_ep_use_zcpy_rx_impl(struct efa_resource *resource,
ofi_hmem_disable_p2p = cuda_p2p_disabled;

efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, false, true, EFA_PROV_NAME);
resource->hints, false, true);

/* System memory P2P should always be enabled */
assert_true(g_efa_hmem_info[FI_HMEM_SYSTEM].initialized);
Expand Down Expand Up @@ -1314,7 +1314,7 @@ void test_efa_rdm_ep_rx_refill_impl(struct efa_resource **state, int threshold,
assert_non_null(resource->hints);
resource->hints->rx_attr->size = rx_size;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
assert_int_equal(efa_rdm_ep_get_rx_pool_size(efa_rdm_ep), rx_size);
Expand Down
Loading