Skip to content

Commit

Permalink
smsc/accelerator: add new smsc component
Browse files Browse the repository at this point in the history
- add smsc component based on the accelerator API IPC functionality.
- adjust smsc base selection logic to enable a pass-through mechanism
  with the new smsc/accelerator component

Signed-off-by: Edgar Gabriel <[email protected]>
  • Loading branch information
edgargabriel committed May 2, 2024
1 parent 7495db7 commit 1e72876
Show file tree
Hide file tree
Showing 9 changed files with 579 additions and 14 deletions.
4 changes: 4 additions & 0 deletions opal/mca/btl/sm/btl_sm_component.c
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,10 @@ mca_btl_sm_component_init(int *num_btls, bool enable_progress_threads, bool enab
mca_btl_sm.super.btl_get = mca_btl_sm_get;
mca_btl_sm.super.btl_put = mca_btl_sm_put;

if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_ACCELERATOR)) {
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_GET;
}

mca_btl_sm.super.btl_bandwidth = 40000; /* Mbs */

if (mca_smsc_base_has_feature(MCA_SMSC_FEATURE_CAN_MAP)) {
Expand Down
58 changes: 58 additions & 0 deletions opal/mca/smsc/accelerator/Makefile.am
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
# reserved.
# Copyright (c) 2017 IBM Corporation. All rights reserved.
# Copyright (c) 2020-2021 Google, LLC. All rights reserved.
# Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#

EXTRA_DIST = post_configure.sh

AM_CPPFLAGS = $(smsc_accelerator_CPPFLAGS)


libmca_smsc_accelerator_la_sources = \
smsc_accelerator_component.c \
smsc_accelerator_module.c \
smsc_accelerator_internal.h \
smsc_accelerator.h

# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).

if MCA_BUILD_opal_smsc_accelerator_DSO
component_noinst =
component_install = mca_smsc_accelerator.la
else
component_noinst = libmca_smsc_accelerator.la
component_install =
endif

mcacomponentdir = $(opallibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
mca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)
mca_smsc_accelerator_la_LIBADD = $(top_builddir)/opal/lib@[email protected] \
$(smsc_accelerator_LIBS)

noinst_LTLIBRARIES = $(component_noinst)
libmca_smsc_accelerator_la_SOURCES = $(libmca_smsc_accelerator_la_sources)
libmca_smsc_accelerator_la_LIBADD = $(smsc_accelerator_LIBS)
libmca_smsc_accelerator_la_LDFLAGS = -module -avoid-version $(smsc_accelerator_LDFLAGS)
34 changes: 34 additions & 0 deletions opal/mca/smsc/accelerator/smsc_accelerator.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/

#ifndef OPAL_MCA_SMSC_ACCELERATOR_H
#define OPAL_MCA_SMSC_ACCELERATOR_H

#include "opal_config.h"

#include "opal/mca/smsc/smsc.h"

mca_smsc_endpoint_t *mca_smsc_accelerator_get_endpoint(opal_proc_t *peer_proc);
void mca_smsc_accelerator_return_endpoint(mca_smsc_endpoint_t *endpoint);

int mca_smsc_accelerator_copy_to(mca_smsc_endpoint_t *endpoint, void *local_address, void *remote_address,
size_t size, void *reg_handle);

int mca_smsc_accelerator_copy_from(mca_smsc_endpoint_t *endpoint, void *local_address,
void *remote_address, size_t size, void *reg_handle);

void *mca_smsc_accelerator_map_peer_region(mca_smsc_endpoint_t *endpoint, uint64_t flags,
void *remote_ptr, size_t size, void **local_ptr);
void mca_smsc_accelerator_unmap_peer_region(void *ctx);

void *mca_smsc_accelerator_register_region(void *local_address, size_t size);
void mca_smsc_accelerator_deregister_region(void *reg_data);

#endif /* OPAL_MCA_SMSC_ACCELERATOR__H */
133 changes: 133 additions & 0 deletions opal/mca/smsc/accelerator/smsc_accelerator_component.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "opal_config.h"

#include "opal/mca/smsc/base/base.h"
#include "opal/mca/smsc/accelerator/smsc_accelerator_internal.h"
#include "opal/mca/accelerator/accelerator.h"
#include "opal/mca/accelerator/base/base.h"

static int mca_smsc_accelerator_component_register(void);
static int mca_smsc_accelerator_component_open(void);
static int mca_smsc_accelerator_component_close(void);
static int mca_smsc_accelerator_component_query(void);
static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void);

#define MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY 80
static const int mca_smsc_accelerator_default_priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY;

mca_smsc_accelerator_component_t mca_smsc_accelerator_component = {
.super = {
.smsc_version = {
MCA_SMSC_DEFAULT_VERSION("accelerator"),
.mca_open_component = mca_smsc_accelerator_component_open,
.mca_close_component = mca_smsc_accelerator_component_close,
.mca_register_component_params = mca_smsc_accelerator_component_register,
},
.priority = MCA_SMSC_ACCELERATOR_DEFAULT_PRIORITY,
.query = mca_smsc_accelerator_component_query,
.enable = mca_smsc_accelerator_component_enable,
},
};

static int mca_smsc_accelerator_component_register(void)
{
mca_smsc_base_register_default_params(&mca_smsc_accelerator_component.super,
mca_smsc_accelerator_default_priority);
return OPAL_SUCCESS;
}

static int mca_smsc_accelerator_component_open(void)
{
return OPAL_SUCCESS;
}

static int mca_smsc_accelerator_component_close(void)
{
if (mca_smsc_accelerator_module.rcache) {
(void) mca_rcache_base_module_destroy(mca_smsc_accelerator_module.rcache);
mca_smsc_accelerator_module.rcache = NULL;
}

return OPAL_SUCCESS;
}

static int mca_smsc_accelerator_component_query(void)
{
if (0 == strcmp(opal_accelerator_base_selected_component.base_version.mca_component_name,
"null")) {
opal_output_verbose(10, opal_smsc_base_framework.framework_output,
"smsc:accelerator:component_query: accelerator component is null: disqualifying myself");
return OPAL_ERROR;
}

if (!opal_accelerator_base_selected_component.is_ipc_enabled()) {
opal_output_verbose(10, opal_smsc_base_framework.framework_output,
"smsc:accelerator:component_query: accelerator component does not have support for IPC operations: disqualifying myself");
return OPAL_ERROR;
}

return OPAL_SUCCESS;
}


static int mca_smsc_accelerator_reg(void *reg_data, void *local_address, size_t size,
mca_rcache_base_registration_t *reg_handle)
{
mca_smsc_accelerator_registration_handle_t *reg = (mca_smsc_accelerator_registration_handle_t *) reg_handle;
int ret;

ret = opal_accelerator.get_ipc_handle (mca_smsc_accelerator_module.device_id, local_address,
&reg->ipc_handle);
memcpy (reg->data.handle.accelerator, reg->ipc_handle.handle, SMSC_ACCELERATOR_HANDLE_SIZE);
if (OPAL_SUCCESS != ret ) {
return OPAL_ERROR;
}

return OPAL_SUCCESS;
}

static int mca_smsc_accelerator_dereg(void *reg_data, mca_rcache_base_registration_t *reg_handle)
{
mca_smsc_accelerator_registration_handle_t *reg = (mca_smsc_accelerator_registration_handle_t *) reg_handle;

OBJ_DESTRUCT(&(reg->ipc_handle));
return OPAL_SUCCESS;
}

static mca_smsc_module_t *mca_smsc_accelerator_component_enable(void)
{
if (0 > mca_smsc_accelerator_component.super.priority) {
return NULL;
}

mca_rcache_base_resources_t rcache_resources = {.cache_name = "smsc_accelerator",
.reg_data = NULL,
.sizeof_reg = sizeof(
mca_smsc_accelerator_registration_handle_t),
.register_mem = mca_smsc_accelerator_reg,
.deregister_mem = mca_smsc_accelerator_dereg};

mca_smsc_accelerator_module.rcache = mca_rcache_base_module_create("grdma", NULL, &rcache_resources);
if (NULL == mca_smsc_accelerator_module.rcache) {
return NULL;
}

/* Not set. Will initialize later */
mca_smsc_accelerator_module.device_id = MCA_ACCELERATOR_NO_DEVICE_ID;

mca_smsc_accelerator_module.prev_smsc = mca_smsc;
if ((NULL != mca_smsc_accelerator_module.prev_smsc) &&
(mca_smsc_accelerator_module.prev_smsc->features & MCA_SMSC_FEATURE_CAN_MAP)) {
mca_smsc_accelerator_module.super.features |= MCA_SMSC_FEATURE_CAN_MAP;
}

return &mca_smsc_accelerator_module.super;
}
70 changes: 70 additions & 0 deletions opal/mca/smsc/accelerator/smsc_accelerator_internal.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2021 Google, Inc. All rights reserved.
* Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/

#ifndef OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H
#define OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H

#include "opal_config.h"

#include "opal/mca/rcache/base/base.h"
#include "opal/mca/rcache/rcache.h"
#include "opal/mca/smsc/accelerator/smsc_accelerator.h"
#include "opal/mca/accelerator/accelerator.h"

#define SMSC_ACCELERATOR_HANDLE_SIZE IPC_MAX_HANDLE_SIZE
struct mca_smsc_accelerator_registration_data_t {
uint64_t base_addr;
union {
uint8_t accelerator[SMSC_ACCELERATOR_HANDLE_SIZE];
void* host;
} handle;
};
typedef struct mca_smsc_accelerator_registration_data_t mca_smsc_accelerator_registration_data_t;

struct mca_smsc_accelerator_registration_handle_t {
mca_rcache_base_registration_t base;
mca_smsc_accelerator_registration_data_t data;
opal_accelerator_ipc_handle_t ipc_handle;
};
typedef struct mca_smsc_accelerator_registration_handle_t mca_smsc_accelerator_registration_handle_t;
OBJ_CLASS_DECLARATION(mca_smsc_accelerator_registration_handle_t);

#define MCA_SMSC_ACCELERATOR_REG_DATA_TO_HANDLE(data_ptr) \
((mca_smsc_accelerator_registration_handle_t *) ((uintptr_t) data_ptr \
- offsetof(mca_smsc_accelerator_registration_handle_t, \
data)))


struct mca_smsc_accelerator_endpoint_t {
mca_smsc_endpoint_t super;
mca_smsc_endpoint_t *prev_endpoint;
mca_rcache_base_module_t *rcache;
};
typedef struct mca_smsc_accelerator_endpoint_t mca_smsc_accelerator_endpoint_t;
OBJ_CLASS_DECLARATION(mca_smsc_accelerator_endpoint_t);

struct mca_smsc_accelerator_component_t {
mca_smsc_component_t super;
};
typedef struct mca_smsc_accelerator_component_t mca_smsc_accelerator_component_t;

struct mca_smsc_accelerator_module_t {
mca_smsc_module_t super;
mca_smsc_module_t *prev_smsc;
mca_rcache_base_module_t *rcache;
int device_id;
};
typedef struct mca_smsc_accelerator_module_t mca_smsc_accelerator_module_t;

extern mca_smsc_accelerator_module_t mca_smsc_accelerator_module;
extern mca_smsc_accelerator_component_t mca_smsc_accelerator_component;

#endif /* OPAL_MCA_SMSC_ACCELERATOR_INTERNAL_H */
Loading

0 comments on commit 1e72876

Please sign in to comment.