Skip to content

UCP/WIREUP: Support EP reconfiguration for non wired-up scenarios #10337

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 18, 2025
Merged

UCP/WIREUP: Support EP reconfiguration for non wired-up scenarios #10337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/ucp/core/ucp_context.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,11 @@ static ucs_config_field_t ucp_context_config_table[] = {
ucs_offsetof(ucp_context_config_t, max_priority_eps),
UCS_CONFIG_TYPE_UINT},

{"WIREUP_VIA_AM_LANE", "n",
"Use AM lane to send wireup messages",
ucs_offsetof(ucp_context_config_t, wireup_via_am_lane),
UCS_CONFIG_TYPE_BOOL},

{NULL}
};

Expand Down
2 changes: 2 additions & 0 deletions src/ucp/core/ucp_context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ typedef struct ucp_context_config {
uint64_t extra_op_attr_flags;
/* Upper limit to the amount of prioritized endpoints */
unsigned max_priority_eps;
/* Use AM lane to send wireup messages */
int wireup_via_am_lane;
} ucp_context_config_t;


Expand Down
59 changes: 29 additions & 30 deletions src/ucp/core/ucp_ep.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -827,6 +827,7 @@ ucp_ep_create_to_worker_addr(ucp_worker_h worker,
ucp_tl_bitmap_t ep_tl_bitmap;
ucs_status_t status;
ucp_ep_h ep;
int am_need_flush;

/* allocate endpoint */
status = ucp_ep_create_base(worker, ep_init_flags, remote_address->name,
Expand All @@ -837,7 +838,8 @@ ucp_ep_create_to_worker_addr(ucp_worker_h worker,

/* initialize transport endpoints */
status = ucp_wireup_init_lanes(ep, ep_init_flags, local_tl_bitmap,
remote_address, addr_indices);
remote_address, addr_indices,
&am_need_flush);
if (status != UCS_OK) {
goto err_delete;
}
Expand Down Expand Up @@ -1861,40 +1863,39 @@ int ucp_ep_config_lane_is_peer_match(const ucp_ep_config_key_t *key1,
config_lane2->dst_md_index);
}

static ucp_lane_index_t
ucp_ep_config_find_match_lane(const ucp_ep_config_key_t *key1,
ucp_lane_index_t lane1,
const ucp_ep_config_key_t *key2)
ucp_lane_index_t
ucp_ep_config_find_match_lane(const ucp_ep_config_key_t *old_key,
ucp_lane_index_t old_lane,
const ucp_ep_config_key_t *new_key)
{
ucp_lane_index_t lane_idx;
ucp_lane_index_t new_lane;

for (lane_idx = 0; lane_idx < key2->num_lanes; ++lane_idx) {
if (ucp_ep_config_lane_is_peer_match(key1, lane1, key2, lane_idx)) {
return lane_idx;
for (new_lane = 0; new_lane < new_key->num_lanes; ++new_lane) {
if (ucp_ep_config_lane_is_peer_match(old_key, old_lane, new_key,
new_lane)) {
return new_lane;
}
}

return UCP_NULL_LANE;
}

static ucp_lane_index_t
ucp_ep_config_find_reusable_lane(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2, ucp_ep_h ep,
const ucp_unpacked_address_t *remote_address,
const unsigned *addr_indices,
ucp_lane_index_t old_lane)
static ucp_lane_index_t ucp_ep_config_find_reusable_lane(
const ucp_ep_config_key_t *old_key, const ucp_ep_config_key_t *new_key,
ucp_ep_h ep, const ucp_unpacked_address_t *remote_address,
const unsigned *addr_indices, ucp_lane_index_t old_lane)
{
ucp_context_h context = ep->worker->context;
ucp_rsc_index_t rsc_index = key1->lanes[old_lane].rsc_index;
ucp_rsc_index_t rsc_index = old_key->lanes[old_lane].rsc_index;
ucp_lane_index_t new_lane;
unsigned addr_index;
const ucp_address_entry_t *ae;

if (old_lane == key1->cm_lane) {
return key2->cm_lane;
if (old_lane == old_key->cm_lane) {
return new_key->cm_lane;
}

new_lane = ucp_ep_config_find_match_lane(key1, old_lane, key2);
new_lane = ucp_ep_config_find_match_lane(old_key, old_lane, new_key);
if (new_lane == UCP_NULL_LANE) {
/* No matching lane was found */
return UCP_NULL_LANE;
Expand All @@ -1921,26 +1922,24 @@ ucp_ep_config_find_reusable_lane(const ucp_ep_config_key_t *key1,

/* Go through the first configuration and check if the lanes selected
* for this configuration could be used for the second configuration */
void ucp_ep_config_lanes_intersect(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2,
void ucp_ep_config_lanes_intersect(const ucp_ep_config_key_t *old_key,
const ucp_ep_config_key_t *new_key,
const ucp_ep_h ep,
const ucp_unpacked_address_t *remote_address,
const unsigned *addr_indices,
ucp_lane_index_t *lane_map)
{
ucp_lane_index_t lane1_idx;
ucp_lane_index_t old_lane;

for (lane1_idx = 0; lane1_idx < key1->num_lanes; ++lane1_idx) {
lane_map[lane1_idx] = ucp_ep_config_find_reusable_lane(key1, key2, ep,
remote_address,
addr_indices,
lane1_idx);
for (old_lane = 0; old_lane < old_key->num_lanes; ++old_lane) {
lane_map[old_lane] = ucp_ep_config_find_reusable_lane(
old_key, new_key, ep, remote_address, addr_indices, old_lane);
}
}

static int ucp_ep_config_lane_is_equal(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2,
ucp_lane_index_t lane)
int ucp_ep_config_lane_is_equal(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2,
ucp_lane_index_t lane)
{
const ucp_ep_config_key_lane_t *config_lane1 = &key1->lanes[lane];
const ucp_ep_config_key_lane_t *config_lane2 = &key2->lanes[lane];
Expand Down
13 changes: 11 additions & 2 deletions src/ucp/core/ucp_ep.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,13 +754,22 @@ int ucp_ep_config_lane_is_peer_match(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2,
ucp_lane_index_t lane2);

void ucp_ep_config_lanes_intersect(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2,
ucp_lane_index_t
ucp_ep_config_find_match_lane(const ucp_ep_config_key_t *old_key,
ucp_lane_index_t old_lane,
const ucp_ep_config_key_t *new_key);

void ucp_ep_config_lanes_intersect(const ucp_ep_config_key_t *old_key,
const ucp_ep_config_key_t *new_key,
const ucp_ep_h ep,
const ucp_unpacked_address_t *remote_address,
const unsigned *addr_indices,
ucp_lane_index_t *lane_map);

int ucp_ep_config_lane_is_equal(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2,
ucp_lane_index_t lane);

int ucp_ep_config_is_equal(const ucp_ep_config_key_t *key1,
const ucp_ep_config_key_t *key2);

Expand Down
10 changes: 8 additions & 2 deletions src/ucp/wireup/select.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2044,7 +2044,8 @@ ucp_wireup_select_wireup_msg_lane(ucp_worker_h worker,
unsigned ep_init_flags,
const ucp_address_entry_t *address_list,
const ucp_wireup_lane_desc_t *lane_descs,
ucp_lane_index_t num_lanes)
ucp_lane_index_t num_lanes,
ucp_lane_index_t am_lane)
{
ucp_context_h context = worker->context;
ucp_lane_index_t p2p_lane = UCP_NULL_LANE;
Expand All @@ -2055,6 +2056,11 @@ ucp_wireup_select_wireup_msg_lane(ucp_worker_h worker,
ucp_lane_index_t lane;
unsigned addr_index;

if (context->config.ext.wireup_via_am_lane) {
ucs_assert(am_lane != UCP_NULL_LANE);
return am_lane;
}

ucp_wireup_fill_aux_criteria(&criteria, ep_init_flags,
UCP_ADDR_IFACE_FLAG_CB_ASYNC);
for (lane = 0; lane < num_lanes; ++lane) {
Expand Down Expand Up @@ -2440,7 +2446,7 @@ ucp_wireup_construct_lanes(const ucp_wireup_select_params_t *select_params,
select_ctx),
select_params->address->address_list,
select_ctx->lane_descs,
key->num_lanes);
key->num_lanes, key->am_lane);
}

for (i = 0; key->rma_bw_lanes[i] != UCP_NULL_LANE; i++) {
Expand Down
Loading
Loading