From b43b3028dc35bb75b96061bee7742496838445ce Mon Sep 17 00:00:00 2001 From: Min Si Date: Thu, 6 May 2021 16:42:50 -0500 Subject: [PATCH 1/3] setup: should not use PMPI in set_mpit_cvar --- src/internal/setup_impl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/setup_impl.c b/src/internal/setup_impl.c index ab30ee5..a0ec83a 100644 --- a/src/internal/setup_impl.c +++ b/src/internal/setup_impl.c @@ -705,7 +705,7 @@ static void set_mpit_cvar(const char *cvar_name, const void *val) /* TODO: Add other data types when needed. */ int val_read = 0; - OSHMPI_CALLMPI(PMPI_T_cvar_write(handle, val)); + OSHMPI_CALLMPI(MPI_T_cvar_write(handle, val)); OSHMPI_CALLMPI(MPI_T_cvar_read(handle, &val_read)); OSHMPI_DBGMSG("MPI_T setup: %s = %d\n", cvar_name, val_read); From 0656b4c9954de8bb89192d82e5239b97a3e0f9aa Mon Sep 17 00:00:00 2001 From: Min Si Date: Fri, 7 May 2021 13:57:33 -0500 Subject: [PATCH 2/3] setup: add an env var to optionally disable MPI_T usage Set OSHMPI_ENABLE_MPI_T=0 to disable MPI_T usage in OSHMPI. It is 1 (enabled) by default --- src/include/oshmpi_impl.h | 3 +++ src/internal/setup_impl.c | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/include/oshmpi_impl.h b/src/include/oshmpi_impl.h index 10f16ef..8693c5b 100644 --- a/src/include/oshmpi_impl.h +++ b/src/include/oshmpi_impl.h @@ -279,6 +279,9 @@ typedef struct { * Invalid when OSHMPI_DISABLE_AM_ASYNC_THREAD is set. * Default value is 1 if either AMO or RMA is AM based; * otherwise 0.*/ + unsigned int enable_mpit; /* OSHMPI_ENABLE_MPI_T: Control mpi_t utilization at shmem_init. + * Value: 0 or 1. Default is 1, enables MPI_T setting. + * Set OSHMPI_ENABLE_MPI_T=0 to disable. */ uint32_t mpi_gpu_features; /* OSHMPI_MPI_GPU_FEATURES: Arbitrary combination with bit shift defined in * OSHMPI_mpi_gpu_feature_shift_t. none and all are two special values. */ #ifndef OSHMPI_DISABLE_DEBUG diff --git a/src/internal/setup_impl.c b/src/internal/setup_impl.c index a0ec83a..35a57c7 100644 --- a/src/internal/setup_impl.c +++ b/src/internal/setup_impl.c @@ -576,6 +576,7 @@ static void print_env(void) " OSHMPI_AMO_OPS %s\n" " OSHMPI_ENABLE_ASYNC_THREAD %d\n" " OSHMPI_MPI_GPU_FEATURES %s\n" + " OSHMPI_ENABLE_MPI_T %d\n" #ifndef OSHMPI_DISABLE_DEBUG " (Invalid options if OSHMPI is built with --enable-fast=ndebug)\n" " OSHMPI_AMO_DBG_MODE %s\n" @@ -583,7 +584,8 @@ static void print_env(void) #endif ,OSHMPI_env.verbose, amo_ops_str, OSHMPI_env.enable_async_thread, - mpi_gpu_features_str + mpi_gpu_features_str, + OSHMPI_env.enable_mpit #ifndef OSHMPI_DISABLE_DEBUG ,getstr_env_dbg_mode(OSHMPI_env.amo_dbg_mode) ,getstr_env_dbg_mode(OSHMPI_env.rma_dbg_mode) @@ -682,6 +684,11 @@ static void initialize_env(void) if (OSHMPI_env.enable_async_thread != 0) OSHMPI_env.enable_async_thread = 1; #endif + + OSHMPI_env.enable_mpit = 1; + val = getenv("OSHMPI_ENABLE_MPI_T"); + if (val && strlen(val) && atoi(val) == 0) + OSHMPI_env.enable_mpit = 0; } static void set_mpit_cvar(const char *cvar_name, const void *val) @@ -714,6 +721,9 @@ static void set_mpit_cvar(const char *cvar_name, const void *val) static void initialize_mpit(void) { + if (!OSHMPI_env.enable_mpit) + return; + int val = 1; set_mpit_cvar("MPIR_CVAR_CH4_RMA_ENABLE_DYNAMIC_AM_PROGRESS", &val); } From 0f53b3bf1fa609e06ca2b1b198422478df26cf35 Mon Sep 17 00:00:00 2001 From: Min Si Date: Fri, 7 May 2021 16:30:14 -0500 Subject: [PATCH 3/3] setup: add debug msg to print cnt of inited cvars --- src/internal/setup_impl.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/internal/setup_impl.c b/src/internal/setup_impl.c index 35a57c7..2c0e21d 100644 --- a/src/internal/setup_impl.c +++ b/src/internal/setup_impl.c @@ -691,7 +691,7 @@ static void initialize_env(void) OSHMPI_env.enable_mpit = 0; } -static void set_mpit_cvar(const char *cvar_name, const void *val) +static int set_mpit_cvar(const char *cvar_name, const void *val) { int mpi_errno = MPI_SUCCESS; @@ -699,12 +699,12 @@ static void set_mpit_cvar(const char *cvar_name, const void *val) char *env_var = NULL; env_var = getenv(cvar_name); if (env_var && strlen(env_var)) - return; + return 0; int cvar_index; OSHMPI_CALLMPI_RET(mpi_errno, MPI_T_cvar_get_index(cvar_name, &cvar_index)); if (mpi_errno == MPI_T_ERR_INVALID_NAME) - return; /* Support of a CVAR is implementation specific */ + return 0; /* Support of a CVAR is implementation specific */ MPI_T_cvar_handle handle; int count; @@ -717,6 +717,7 @@ static void set_mpit_cvar(const char *cvar_name, const void *val) OSHMPI_DBGMSG("MPI_T setup: %s = %d\n", cvar_name, val_read); MPI_T_cvar_handle_free(&handle); + return 1; } static void initialize_mpit(void) @@ -724,8 +725,10 @@ static void initialize_mpit(void) if (!OSHMPI_env.enable_mpit) return; - int val = 1; - set_mpit_cvar("MPIR_CVAR_CH4_RMA_ENABLE_DYNAMIC_AM_PROGRESS", &val); + int val = 1, cnt = 0; + cnt += set_mpit_cvar("MPIR_CVAR_CH4_RMA_ENABLE_DYNAMIC_AM_PROGRESS", &val); + + OSHMPI_DBGMSG("Initialized %d MPI_T CVAR variables\n", cnt); } void OSHMPI_initialize_thread(int required, int *provided)