From defea5f18103e4de2b3c34d90bf1f5124a3c8340 Mon Sep 17 00:00:00 2001 From: Aurelien Bouteiller Date: Fri, 25 Apr 2025 11:01:13 -0400 Subject: [PATCH 1/5] bugfix: Setting OMPI_MPI_THREAD_LEVEL to a value different than `requested` in `MPI_Init_thread` would invoke the error handler, even though it is an useful override in some threaded library use cases. Signed-off-by: Aurelien Bouteiller (cherry picked from commit 27332fcc80a03e996bf8d1ed231690ee6eacdd4e) --- ompi/mpi/c/init.c | 8 +++++++- ompi/mpi/c/init_thread.c | 32 ++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/ompi/mpi/c/init.c b/ompi/mpi/c/init.c index eb5a50a7643..26cee04ff4f 100644 --- a/ompi/mpi/c/init.c +++ b/ompi/mpi/c/init.c @@ -13,6 +13,7 @@ * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -54,7 +55,12 @@ int MPI_Init(int *argc, char ***argv) if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) { required = atoi(env); - if (required < MPI_THREAD_SINGLE || required > MPI_THREAD_MULTIPLE) { + /* In the future we may have to contend with non-sequential (MPI ABI) values + * If you are implementing MPI ABI changes please refer to + * https://github.com/open-mpi/ompi/pull/13211#discussion_r2085086844 + */ + if (required != MPI_THREAD_SINGLE && required != MPI_THREAD_FUNNELED && + required != MPI_THREAD_SERIALIZED && required != MPI_THREAD_MULTIPLE) { required = MPI_THREAD_MULTIPLE; } } diff --git a/ompi/mpi/c/init_thread.c b/ompi/mpi/c/init_thread.c index 95ca9df25e2..581edf9ff45 100644 --- a/ompi/mpi/c/init_thread.c +++ b/ompi/mpi/c/init_thread.c @@ -16,6 +16,7 @@ * Copyright (c) 2015-2018 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -48,6 +49,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required, int *provided) { int err, safe_required = MPI_THREAD_SERIALIZED; + bool err_arg_required = false; char *env; ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided); @@ -55,14 +57,28 @@ int MPI_Init_thread(int *argc, char ***argv, int required, /* Detect an incorrect thread support level, but dont report until we have the minimum * infrastructure setup. */ - if( (MPI_THREAD_SINGLE == required) || (MPI_THREAD_SERIALIZED == required) || - (MPI_THREAD_FUNNELED == required) || (MPI_THREAD_MULTIPLE == required) ) { + err_arg_required = (required != MPI_THREAD_SINGLE && required != MPI_THREAD_FUNNELED && + required != MPI_THREAD_SERIALIZED && required != MPI_THREAD_MULTIPLE); + if (!err_arg_required) { + safe_required = required; + } - if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) { - safe_required = atoi(env); - } - else { - safe_required = required; + /* check for environment overrides for required thread level. If + * there is, check to see that it is a valid/supported thread level. + * If valid, the environment variable always override the provided thread + * level (even if lower than argument `required`). A user program can + * check `provided != required` to check if `required` has been overruled. + */ + if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) { + int env_required = atoi(env); + /* In the future we may have to contend with non-sequential (MPI ABI) values + * If you are implementing MPI ABI changes please refer to + * https://github.com/open-mpi/ompi/pull/13211#discussion_r2085086844 + */ + err_arg_required |= (env_required != MPI_THREAD_SINGLE && env_required != MPI_THREAD_FUNNELED && + env_required != MPI_THREAD_SERIALIZED && env_required != MPI_THREAD_MULTIPLE); + if (!err_arg_required) { + safe_required = env_required; } } @@ -78,7 +94,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required, err = ompi_mpi_init(0, NULL, safe_required, provided, false); } - if( safe_required != required ) { + if (err_arg_required) { /* Trigger the error handler for the incorrect argument. Keep it separate from the * check on the ompi_mpi_init return and report a nice, meaningful error message to * the user. */ From 6ad6cc8745462458b6f072aa7d2c102f432682a7 Mon Sep 17 00:00:00 2001 From: Aurelien Bouteiller Date: Thu, 12 Jun 2025 23:42:06 -0400 Subject: [PATCH 2/5] OMPI_MPI_THREAD_LEVEL can now take 'multiple' 'MPI_THREAD_MULTIPLE' (single,etc) in addition to numeric 0-3 values Signed-off-by: Aurelien Bouteiller (cherry picked from commit 3de248902a38a7ebe8299f8a35e677eb8196b255) --- ompi/mpi/c/init.c | 14 ++------------ ompi/mpi/c/init_thread.c | 16 +++------------- ompi/runtime/mpiruntime.h | 22 +++++++++++++++++++++ ompi/runtime/ompi_mpi_init.c | 37 ++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 25 deletions(-) diff --git a/ompi/mpi/c/init.c b/ompi/mpi/c/init.c index 26cee04ff4f..290a28efd5b 100644 --- a/ompi/mpi/c/init.c +++ b/ompi/mpi/c/init.c @@ -46,23 +46,13 @@ int MPI_Init(int *argc, char ***argv) { int err; int provided; - char *env; int required = MPI_THREAD_SINGLE; /* check for environment overrides for required thread level. If there is, check to see that it is a valid/supported thread level. If not, default to MPI_THREAD_MULTIPLE. */ - - if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) { - required = atoi(env); - /* In the future we may have to contend with non-sequential (MPI ABI) values - * If you are implementing MPI ABI changes please refer to - * https://github.com/open-mpi/ompi/pull/13211#discussion_r2085086844 - */ - if (required != MPI_THREAD_SINGLE && required != MPI_THREAD_FUNNELED && - required != MPI_THREAD_SERIALIZED && required != MPI_THREAD_MULTIPLE) { - required = MPI_THREAD_MULTIPLE; - } + if (OMPI_SUCCESS > ompi_getenv_mpi_thread_level(&required)) { + required = MPI_THREAD_MULTIPLE; } /* Call the back-end initialization function (we need to put as diff --git a/ompi/mpi/c/init_thread.c b/ompi/mpi/c/init_thread.c index 581edf9ff45..a12743ffb3b 100644 --- a/ompi/mpi/c/init_thread.c +++ b/ompi/mpi/c/init_thread.c @@ -50,12 +50,13 @@ int MPI_Init_thread(int *argc, char ***argv, int required, { int err, safe_required = MPI_THREAD_SERIALIZED; bool err_arg_required = false; - char *env; ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided); /* Detect an incorrect thread support level, but dont report until we have the minimum * infrastructure setup. + * In the future integer MPI_ABI values for MPI_THREAD_SINGLE-MULTIPLE + * may have gaps between them, so just checking the range is not enough. */ err_arg_required = (required != MPI_THREAD_SINGLE && required != MPI_THREAD_FUNNELED && required != MPI_THREAD_SERIALIZED && required != MPI_THREAD_MULTIPLE); @@ -69,18 +70,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required, * level (even if lower than argument `required`). A user program can * check `provided != required` to check if `required` has been overruled. */ - if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) { - int env_required = atoi(env); - /* In the future we may have to contend with non-sequential (MPI ABI) values - * If you are implementing MPI ABI changes please refer to - * https://github.com/open-mpi/ompi/pull/13211#discussion_r2085086844 - */ - err_arg_required |= (env_required != MPI_THREAD_SINGLE && env_required != MPI_THREAD_FUNNELED && - env_required != MPI_THREAD_SERIALIZED && env_required != MPI_THREAD_MULTIPLE); - if (!err_arg_required) { - safe_required = env_required; - } - } + err_arg_required |= (OMPI_SUCCESS > ompi_getenv_mpi_thread_level(&safe_required)); *provided = safe_required; diff --git a/ompi/runtime/mpiruntime.h b/ompi/runtime/mpiruntime.h index 5ce1ce53539..6fde46fb190 100644 --- a/ompi/runtime/mpiruntime.h +++ b/ompi/runtime/mpiruntime.h @@ -16,6 +16,7 @@ * Copyright (c) 2009 University of Houston. All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. * Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. + * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -163,6 +164,27 @@ extern opal_hash_table_t ompi_mpi_f90_complex_hashtable; /** version string of ompi */ OMPI_DECLSPEC extern const char ompi_version_string[]; +/** + * Obtain the required thread level from environment (if any) + * + * @param requested Thread support that is requested (OUT) + * + * @returns Error code if environment exist but has an invalid value + * + * The function reads the environment variable OMPI_MPI_THREAD_LEVEL + * and set parameter requested accordingly. If the environment is not + * set, or has an invalid value, requested is left unchanged. + */ +int ompi_getenv_mpi_thread_level(int *requested); + +/** + * Determine the thread level + * + * @param requested Thread support that is requested (IN) + * @param provided Thread support that is provided (OUT) + */ +void ompi_mpi_thread_level(int requested, int *provided); + /** * Determine the thread level * diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 19c0999d163..787e1e10249 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -29,6 +29,7 @@ * Copyright (c) 2021 Nanook Consulting. All rights reserved. * Copyright (c) 2021-2022 Triad National Security, LLC. All rights * reserved. + * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -268,6 +269,42 @@ MPI_Fint *MPI_F08_STATUSES_IGNORE = NULL; #include "mpif-c-constants.h" +int ompi_getenv_mpi_thread_level(int *requested) +{ + char* env; + if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) { + /* deal with string values, int values (no atoi, it doesn't error check) */ + /* In the future integer MPI_ABI values for MPI_THREAD_SINGLE-MULTIPLE + * may be non-sequential (but ordered) integer values. + * If you are implementing MPI ABI changes please refer to + * https://github.com/open-mpi/ompi/pull/13211#discussion_r2085086844 + */ + if (0 == strcasecmp(env, "multiple") || + 0 == strcasecmp(env, "MPI_THREAD_MULTIPLE") || + 0 == strcmp(env, "3")) { + return *requested = MPI_THREAD_MULTIPLE; + } + if (0 == strcasecmp(env, "serialized") || + 0 == strcasecmp(env, "MPI_THREAD_SERIALIZED") || + 0 == strcmp(env, "2")) { + return *requested = MPI_THREAD_SERIALIZED; + } + if (0 == strcasecmp(env, "funneled") || + 0 == strcasecmp(env, "MPI_THREAD_FUNNELED") || + 0 == strcmp(env, "1")) { + return *requested = MPI_THREAD_FUNNELED; + } + if (0 == strcasecmp(env, "single") || + 0 == strcasecmp(env, "MPI_THREAD_SINGLE") || + 0 == strcmp(env, "0")) { + return *requested = MPI_THREAD_SINGLE; + } + /* the env value is invalid... */ + return OMPI_ERR_BAD_PARAM; + } + return OMPI_SUCCESS; +} + void ompi_mpi_thread_level(int requested, int *provided) { /** From 2d5505cf08c6abc9228efa03c70877ad1380a49d Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Sat, 21 Jun 2025 15:34:04 -0400 Subject: [PATCH 3/5] docs: numerous updates to MPI_Init*/MPI_Finalize*/MPI_Session_* man pages Including, but not limited to: * Added much more description of and distinction between the MPI world model and the MPI session model. Updated a lot of old, pre-MPI-world-model/pre-MPI-session-model text that was now stale / outdated, especially in the following pages: * MPI_Init(3), MPI_Init_thread(3) * MPI_Initialized(3) * MPI_Finalize(3) * MPI_Finalized(3) * MPI_Session_init(3) * MPI_Session_finalize(3) * Numerous formatting updates * Slightly improve the C code examples * Describe the mathematical relationship between the various MPI_THREAD_* constants in MPI_Init_thread(3) * Note that the mathematical relationships render nicely in HTML, but don't render entirely properly in nroff. This commit author is of the opinion that the nroff rendering is currently "good enough", and some Sphinx maintainer will fix it someday. * Add descriptions about the $OMPI_MPI_THREAD_LEVEL env variable and how it is used in MPI_Init_thread(3) * Added more seealso links Signed-off-by: Jeff Squyres (cherry picked from commit aff3afde47bdc30643c0b5e97833793910c54999) --- docs/man-openmpi/man3/MPI_Finalize.3.rst | 114 ++++++++----- docs/man-openmpi/man3/MPI_Finalized.3.rst | 25 ++- docs/man-openmpi/man3/MPI_Init.3.rst | 56 ++++-- docs/man-openmpi/man3/MPI_Init_thread.3.rst | 160 +++++++++++++----- docs/man-openmpi/man3/MPI_Initialized.3.rst | 17 +- .../man3/MPI_Session_finalize.3.rst | 16 +- docs/man-openmpi/man3/MPI_Session_init.3.rst | 46 ++++- 7 files changed, 309 insertions(+), 125 deletions(-) diff --git a/docs/man-openmpi/man3/MPI_Finalize.3.rst b/docs/man-openmpi/man3/MPI_Finalize.3.rst index 174b0c77ebb..032eb7fd40f 100644 --- a/docs/man-openmpi/man3/MPI_Finalize.3.rst +++ b/docs/man-openmpi/man3/MPI_Finalize.3.rst @@ -5,7 +5,7 @@ MPI_Finalize .. include_body -:ref:`MPI_Finalize` |mdash| Terminates MPI execution environment. +:ref:`MPI_Finalize` |mdash| Terminates MPI world model. SYNTAX ------ @@ -48,56 +48,82 @@ OUTPUT PARAMETER DESCRIPTION ----------- -This routine cleans up all MPI states. Once this routine is called, no -MPI routine (not even MPI_Init) may be called, except for -:ref:`MPI_Get_version`, :ref:`MPI_Initialized`, and :ref:`MPI_Finalized`. Unless there has -been a call to :ref:`MPI_Abort`, you must ensure that all pending -communications involving a process are complete before the process calls -:ref:`MPI_Finalize`. If the call returns, each process may either continue -local computations or exit without participating in further -communication with other processes. At the moment when the last process -calls :ref:`MPI_Finalize`, all pending sends must be matched by a receive, and -all pending receives must be matched by a send. - -:ref:`MPI_Finalize` is collective over all connected processes. If no processes -were spawned, accepted, or connected, then this means it is collective -over MPI_COMM_WORLD. Otherwise, it is collective over the union of all -processes that have been and continue to be connected. +This routine finalizes the MPI world model. If the MPI world model +has been initialized in an MPI process, it *must* be finalized exactly +once by invoking this routine during the lifetime of that MPI process. +This is different than the MPI session model, which can be initialized +and finalized multiple times in an MPI process. See +:ref:`MPI_Session_init` and :ref:`MPI_Session_finalize`. + +Unless there has been a call to :ref:`MPI_Abort`, you must +ensure that all pending communications in the MPI world model +involving a process are complete before the process calls +:ref:`MPI_Finalize`. If the call returns, each process may either +continue local computations or exit without participating in further +communication with other processes in the MPI world model. At the +moment when the last process calls :ref:`MPI_Finalize`, all pending +sends in the MPI world model must be matched by a receive, and all +pending receives in the MPI world model must be matched by a send. + +See `MPI-5.0:11.4.1 `_ for a list of MPI +functionality that is available (e.g., even when the MPI +world model has not yet initialized or has already been finalized). + +:ref:`MPI_Finalize` is collective over all connected processes. If no +processes were spawned, accepted, or connected, then this means it is +collective over ``MPI_COMM_WORLD``. Otherwise, it is collective over +the union of all processes that have been and continue to be +connected. NOTES ----- -All processes must call this routine before exiting. All processes will -still exist but may not make any further MPI calls. :ref:`MPI_Finalize` -guarantees that all local actions required by communications the user -has completed will, in fact, occur before it returns. However, -:ref:`MPI_Finalize` guarantees nothing about pending communications that have -not been completed; completion is ensured only by :ref:`MPI_Wait`, :ref:`MPI_Test`, or -:ref:`MPI_Request_free` combined with some other verification of completion. - -For example, a successful return from a blocking communication operation -or from :ref:`MPI_Wait` or :ref:`MPI_Test` means that the communication is completed -by the user and the buffer can be reused, but does not guarantee that -the local process has no more work to do. Similarly, a successful return -from :ref:`MPI_Request_free` with a request handle generated by an :ref:`MPI_Isend` -nullifies the handle but does not guarantee that the operation has -completed. The :ref:`MPI_Isend` is complete only when a matching receive has -completed. - -If you would like to cause actions to happen when a process finishes, -attach an attribute to MPI_COMM_SELF with a callback function. Then, -when :ref:`MPI_Finalize` is called, it will first execute the equivalent of an -:ref:`MPI_Comm_free` on MPI_COMM_SELF. This will cause the delete callback -function to be executed on all keys associated with MPI_COMM_SELF in an -arbitrary order. If no key has been attached to MPI_COMM_SELF, then no -callback is invoked. This freeing of MPI_COMM_SELF happens before any -other parts of MPI are affected. Calling :ref:`MPI_Finalized` will thus return -"false" in any of these callback functions. Once you have done this with -MPI_COMM_SELF, the results of :ref:`MPI_Finalize` are not specified. +The MPI session model is different than the MPI world model, and has +different scopes of availability for MPI functionality. See +:ref:`MPI_Session_init` and :ref:`MPI_Session_finalize`. + +All processes that initialized the MPI world model must call this +routine before exiting. All processes will still exist but may not +make any further MPI calls in the MPI world model. :ref:`MPI_Finalize` +guarantees that all local actions required by communications in the +MPI world model that the user has completed will, in fact, occur +before it returns. However, :ref:`MPI_Finalize` guarantees nothing +about pending communications in the MPI world model that have not been +completed; completion is ensured only by the :ref:`MPI_Wait` and +:ref:`MPI_Test` variants, or :ref:`MPI_Request_free` combined with +some other verification of completion. + +For example, a successful return from a blocking communication +operation or from one of the :ref:`MPI_Wait` or :ref:`MPI_Test` +varients means that the communication is completed by the user and the +buffer can be reused, but does not guarantee that the local process +has no more work to do. Similarly, a successful return from +:ref:`MPI_Request_free` with a request handle generated by an +:ref:`MPI_Isend` nullifies the handle but does not guarantee that the +operation has completed. The :ref:`MPI_Isend` is complete only when a +matching receive has completed. + +If you would like to cause actions to happen when a process finalizes the MPI +world model, attach an attribute to ``MPI_COMM_SELF`` with a callback +function. Then, when :ref:`MPI_Finalize` is called, it will first +execute the equivalent of an :ref:`MPI_Comm_free` on +``MPI_COMM_SELF``. This will cause the delete callback function to be +executed on all keys associated with ``MPI_COMM_SELF`` in an arbitrary +order. If no key has been attached to ``MPI_COMM_SELF``, then no +callback is invoked. This freeing of ``MPI_COMM_SELF`` happens before +any other parts of the MPI world model are affected. Calling +:ref:`MPI_Finalized` will thus return ``false`` in any of these +callback functions. Once you have done this with ``MPI_COMM_SELF``, +the results of :ref:`MPI_Finalize` are not specified. ERRORS ------ .. include:: ./ERRORS.rst -.. seealso:: :ref:`MPI_Init` +.. seealso:: + * :ref:`MPI_Finalized` + * :ref:`MPI_Init` + * :ref:`MPI_Initialized` + * :ref:`MPI_Session_init` + * :ref:`MPI_Session_finalize` diff --git a/docs/man-openmpi/man3/MPI_Finalized.3.rst b/docs/man-openmpi/man3/MPI_Finalized.3.rst index 84c3e71d7a1..4d1cc0117b0 100644 --- a/docs/man-openmpi/man3/MPI_Finalized.3.rst +++ b/docs/man-openmpi/man3/MPI_Finalized.3.rst @@ -5,7 +5,7 @@ MPI_Finalized .. include_body -:ref:`MPI_Finalized` |mdash| Checks whether MPI has been finalized +:ref:`MPI_Finalized` |mdash| Checks whether the MPI world model has been finalized SYNTAX ------ @@ -45,20 +45,31 @@ Fortran 2008 Syntax OUTPUT PARAMETER ---------------- -* ``flag`` : True if MPI was finalized, and false otherwise (logical). +* ``flag`` : True if the MPI world model was finalized, and false + otherwise (logical). * ``ierror`` : Fortran only: Error status (integer). DESCRIPTION ----------- -This routine may be used to determine whether MPI has been finalized. It -is one of a small number of routines that may be called before MPI is -initialized and after MPI has been finalized (:ref:`MPI_Initialized` is -another). +This routine may be used to determine whether the MPI world model has +been finalized. A different routine |mdash| :ref:`MPI_Initialized` +|mdash| is used to indicate whether the MPI world model has been +initialized. + +See `MPI-5.0:11.4.1 `_ for a list of MPI +functionality that is available (e.g., even when the MPI +world model has not yet initialized or has already been finalized). ERRORS ------ .. include:: ./ERRORS.rst -.. seealso:: :ref:`MPI_Init` +.. seealso:: + * :ref:`MPI_Init` + * :ref:`MPI_Init_thread` + * :ref:`MPI_Finalize` + * :ref:`MPI_Finalized` + * :ref:`MPI_Session_init` + * :ref:`MPI_Session_finalize` diff --git a/docs/man-openmpi/man3/MPI_Init.3.rst b/docs/man-openmpi/man3/MPI_Init.3.rst index 6477ec495c9..08af1fa7d01 100644 --- a/docs/man-openmpi/man3/MPI_Init.3.rst +++ b/docs/man-openmpi/man3/MPI_Init.3.rst @@ -6,7 +6,7 @@ MPI_Init .. include_body -:ref:`MPI_Init` |mdash| Initializes the MPI execution environment +:ref:`MPI_Init` |mdash| Initializes the MPI world model SYNTAX @@ -56,23 +56,40 @@ OUTPUT PARAMETER DESCRIPTION ----------- -This routine, or :ref:`MPI_Init_thread`, must be called before most other MPI -routines are called. There are a small number of errors, such as -:ref:`MPI_Initialized` and :ref:`MPI_Finalized`. MPI can be initialized at most once; -subsequent calls to :ref:`MPI_Init` or :ref:`MPI_Init_thread` are erroneous. +This routine, or :ref:`MPI_Init_thread`, initializes the MPI world +model. Either of these routines must be called before MPI +communication routines are called within the MPI world model. The MPI +world model can be initialized at most exactly once in the lifetime of +an MPI process. This is different than the MPI session model, which +can be initialized and finalized multiple times in an MPI process. +See :ref:`MPI_Session_init` and :ref:`MPI_Session_finalize`. -All MPI programs must contain a call to :ref:`MPI_Init` or :ref:`MPI_Init_thread`. -Open MPI accepts the C *argc* and *argv* arguments to main, but neither -modifies, interprets, nor distributes them: +See `MPI-5.0:11.4.1 `_ for a list of MPI +functionality that is available (e.g., even when the MPI +world model has not yet initialized or has already been finalized). + +Open MPI's :ref:`MPI_Init` and :ref:`MPI_Init_thread` both accept the +C *argc* and *argv* arguments to main, but neither modifies, +interprets, nor distributes them: .. code-block:: c - /* declare variables */ - MPI_Init(&argc, &argv); - /* parse arguments */ - /* main program */ - MPI_Finalize(); + #include + int main(int argv, char *argv[]) { + MPI_Init(&argc, &argv); + /* ...body of main MPI pogram... */ + MPI_Finalize(); + return 0; + } + +By default, :ref:`MPI_Init` is effectively equivalent to invoking +:ref:`MPI_Init_thread` with a *required* value of +``MPI_THREAD_SINGLE``. However, if the ``OMPI_MPI_THREAD_LEVEL`` +environment variable is set to a valid value when :ref:`MPI_Init` is +invoked, it is equivalent to invoking :ref:`MPI_Init_thread` with +*required* set to the corresponding value of the ``OMPI_MPI_THREAD_LEVEL`` +environment variable. See :ref:`MPI_Init_thread` for more details. NOTES ----- @@ -80,11 +97,12 @@ NOTES The Fortran version does not have provisions for *argc* and *argv* and takes only IERROR. -The MPI Standard does not say what a program can do before an :ref:`MPI_Init` -or after an :ref:`MPI_Finalize`. In the Open MPI implementation, it should do -as little as possible. In particular, avoid anything that changes the -external state of the program, such as opening files, reading standard -input, or writing to standard output. +The MPI Standard does not specify what a program using the MPI world +model can do before invoking :ref:`MPI_Init` or :ref:`MPI_Init_thread` +or after invoking :ref:`MPI_Finalize`. In the Open MPI implementation, +it should do as little as possible. In particular, avoid anything that +changes the external state of the program, such as opening files, +reading standard input, or writing to standard output. ERRORS @@ -97,3 +115,5 @@ ERRORS * :ref:`MPI_Initialized` * :ref:`MPI_Finalize` * :ref:`MPI_Finalized` + * :ref:`MPI_Session_finalize` + * :ref:`MPI_Session_init` diff --git a/docs/man-openmpi/man3/MPI_Init_thread.3.rst b/docs/man-openmpi/man3/MPI_Init_thread.3.rst index e5173ff90a4..aca6307a347 100644 --- a/docs/man-openmpi/man3/MPI_Init_thread.3.rst +++ b/docs/man-openmpi/man3/MPI_Init_thread.3.rst @@ -6,7 +6,7 @@ MPI_Init_thread .. include_body -:ref:`MPI_Init_thread` |mdash| Initializes the MPI execution environment +:ref:`MPI_Init_thread` |mdash| Initializes the MPI world model SYNTAX @@ -61,64 +61,142 @@ OUTPUT PARAMETERS DESCRIPTION ----------- -This routine, or :ref:`MPI_Init`, must be called before most other MPI routines -are called. There are a small number of exceptions, such as -:ref:`MPI_Initialized` and :ref:`MPI_Finalized`. MPI can be initialized at most once; -subsequent calls to :ref:`MPI_Init` or :ref:`MPI_Init_thread` are erroneous. +This routine, or :ref:`MPI_Init`, initializes the MPI world +model. Either of these routines must be called before MPI +communication routines are called within the MPI world model. The MPI +world model can be initialized at most exactly once in the lifetime of +an MPI process. This is different than the MPI session model, which +can be initialized and finalized multiple times in an MPI process. +See :ref:`MPI_Session_init` and :ref:`MPI_Session_finalize`. -:ref:`MPI_Init_thread`, as compared to :ref:`MPI_Init`, has a provision to request a -certain level of thread support in *required*: +See `MPI-5.0:11.4.1 `_ for a list of MPI +functionality that is available (e.g., even when the MPI +world model has not yet initialized or has already been finalized). -MPI_THREAD_SINGLE - Only one thread will execute. +The MPI world model can be initialized at most once; subsequent calls +to :ref:`MPI_Init` or :ref:`MPI_Init_thread` are erroneous. -MPI_THREAD_FUNNELED - If the process is multithreaded, only the thread that called - :ref:`MPI_Init_thread` will make MPI calls. +Alternatively, instead of the MPI world model, MPI applications can +use the sessions model; see :ref:`MPI_Session_init`. -MPI_THREAD_SERIALIZED - If the process is multithreaded, only one thread will make MPI - library calls at one time. +Upon return, the level of thread support available to the program is +set in *provided*. In Open MPI, the value is dependent on how the +library was configured and built. Note that there is no guarantee that +*provided* will be greater than or equal to *required*. -MPI_THREAD_MULTIPLE - If the process is multithreaded, multiple threads may call MPI at - once with no restrictions. +Open MPI accepts the C *argc* and *argv* arguments to main, but +neither modifies, interprets, nor distributes them: -The level of thread support available to the program is set in -*provided*. In Open MPI, the value is dependent on how the library was -configured and built. Note that there is no guarantee that *provided* -will be greater than or equal to *required*. +.. code-block:: c -Also note that calling :ref:`MPI_Init_thread` with a *required* value of -MPI_THREAD_SINGLE is equivalent to calling :ref:`MPI_Init`. + #include -All MPI programs must contain a call to :ref:`MPI_Init` or :ref:`MPI_Init_thread`. -Open MPI accepts the C *argc* and *argv* arguments to main, but neither -modifies, interprets, nor distributes them: + int main(int argv, char *argv[]) { + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + /* ...body of main MPI pogram... */ + MPI_Finalize(); + return 0; + } -.. code-block:: c - /* declare variables */ - MPI_Init_thread(&argc, &argv, req, &prov); - /* parse arguments */ - /* main program */ - MPI_Finalize(); +:ref:`MPI_Init_thread` has both a direct and an indirect mechanism to +request a specific level of thread support. :ref:`MPI_Init` only has +an indirect mechanism to request a specific level of thread support. + +Direct request of thread level +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:ref:`MPI_Init_thread` has the *required* parameter, which can be set +to any one of the following constants (from ``mpi.h``): + +* ``MPI_THREAD_SINGLE``: Indicating that only one thread will execute. + +* ``MPI_THREAD_FUNNELED``: Indicating that if the process is + multithreaded, only the thread that called :ref:`MPI_Init_thread` + will make MPI calls. + +* ``MPI_THREAD_SERIALIZED``: Indicating that if the process is + multithreaded, only one thread will make MPI library calls at one + time. + +* ``MPI_THREAD_MULTIPLE``: Indicating that if the process is + multithreaded, multiple threads may call MPI at once with no + restrictions. + +The values of these constants adhere to the following relationships: + +.. math:: + :nowrap: + + \begin{eqnarray} + MPI\_THREAD\_SINGLE & < & MPI\_THREAD\_FUNNELED \\ + MPI\_THREAD\_FUNNELED & < & MPI\_THREAD\_SERIALIZED \\ + MPI\_THREAD\_SERIALIZED & < & MPI\_THREAD\_MULTIPLE \\ + \end{eqnarray} + +Indirect request of thread level +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Both :ref:`MPI_Init_thread` and :ref:`MPI_Init` support an indirect +method of indicating the required thread level: setting the +``OMPI_MPI_THREAD_LEVEL`` environment variable: + +* If the ``OMPI_MPI_THREAD_LEVEL`` environment variable is set at the + time :ref:`MPI_Init` is invoked, it behaves as if + :ref:`MPI_Init_thread` was invoked with the corresponding + ``MPI_THREAD_*`` constant value passed via the *required* parameter. + +* If the ``OMPI_MPI_THREAD_LEVEL`` environment variable is set at the + time :ref:`MPI_Init_thread` is invoked, the ``MPI_THREAD_*`` + constant value corresponding to the environment variable value + overrides the value passed via the *required* parameter. + +The ``OMPI_MPI_THREAD_LEVEL`` environment variable can be set to any +of the values listed below. + +.. list-table:: + :header-rows: 1 + + * - Value that Open MPI uses + - Allowable values (case-insensitive) + + * - ``MPI_THREAD_SINGLE`` + - ``MPI_THREAD_SINGLE``, ``SINGLE``, 0 + + * - ``MPI_THREAD_FUNNELED`` + - ``MPI_THREAD_FUNNELED``, ``FUNNELED``, 1 + + * - ``MPI_THREAD_SERIALIZED`` + - ``MPI_THREAD_SERIALIZED``, ``SERIALIZED``, 2 + + * - ``MPI_THREAD_MULTIPLE`` + - ``MPI_THREAD_MULTIPLE``, ``MULTIPLE``, 3 + +.. note:: Prior to Open MPI v6.0.0, only the integer values 0 through + 3 were acceptable values for the ``OMPI_MPI_THREAD_LEVEL`` + environment variable. + Starting with Open MPI v6.0.0, the Open MPI community + recomends using one of the string name variants so that it + can be correctly mapped to the corresponding Open MPI ABI + value or the MPI Standard ABI value, as relevant. NOTES ----- The Fortran version does not have provisions for ``argc`` and ``argv`` and -takes only ``IERROR``. +takes only ``REQUIRED``, ``PROVIDED``, and ``IERROR``. It is the caller's responsibility to check the value of ``provided``, as it may be less than what was requested in ``required``. -The MPI Standard does not say what a program can do before an -:ref:`MPI_Init_thread` or after an :ref:`MPI_Finalize`. In the Open MPI -implementation, it should do as little as possible. In particular, avoid -anything that changes the external state of the program, such as opening -files, reading standard input, or writing to standard output. +The MPI Standard does not specify what a program using the MPI world +model can do before invoking :ref:`MPI_Init` or :ref:`MPI_Init_thread` +or after invoking :ref:`MPI_Finalize`. In the Open MPI implementation, +it should do as little as possible. In particular, avoid anything that +changes the external state of the program, such as opening files, +reading standard input, or writing to standard output. MPI_THREAD_MULTIPLE Support @@ -129,7 +207,7 @@ Open MPI was built supports threading. You can check the output of :ref:`ompi_info(1) ` to see if Open MPI has ``MPI_THREAD_MULTIPLE`` support: -:: +.. code-block:: bash shell$ ompi_info | grep "Thread support" Thread support: posix (MPI_THREAD_MULTIPLE: yes, OPAL support: yes, OMPI progress: no, Event lib: yes) @@ -153,3 +231,5 @@ ERRORS * :ref:`MPI_Initialized` * :ref:`MPI_Finalize` * :ref:`MPI_Finalized` + * :ref:`MPI_Session_finalize` + * :ref:`MPI_Session_init` diff --git a/docs/man-openmpi/man3/MPI_Initialized.3.rst b/docs/man-openmpi/man3/MPI_Initialized.3.rst index 069d089c7dc..46ccf0ba5b8 100644 --- a/docs/man-openmpi/man3/MPI_Initialized.3.rst +++ b/docs/man-openmpi/man3/MPI_Initialized.3.rst @@ -6,7 +6,7 @@ MPI_Initialized .. include_body -:ref:`MPI_Initialized` |mdash| Checks whether MPI has been initialized +:ref:`MPI_Initialized` |mdash| Checks whether the MPI world model has been initialized SYNTAX @@ -48,15 +48,20 @@ Fortran 2008 Syntax OUTPUT PARAMETERS ----------------- -* ``flag``: True if MPI has been initialized, and false otherwise (logical). +* ``flag``: True if the MPI world model has been initialized, and false otherwise (logical). * ``ierror``: Fortran only: Error status (integer). DESCRIPTION ----------- -This routine may be used to determine whether MPI has been initialized. -It is one of a small number of routines that may be called before MPI is -initialized and after MPI has been finalized (:ref:`MPI_Finalized` is another). +This routine may be used to determine whether the MPI world model has +been initialized. A different routine |mdash| :ref:`MPI_Finalized` +|mdash| is used to indicate whether the MPI world model has been +finalized. + +See `MPI-5.0:11.4.1 `_ for a list of MPI +functionality that is available (e.g., even when the MPI +world model has not yet initialized or has already been finalized). ERRORS @@ -69,3 +74,5 @@ ERRORS * :ref:`MPI_Init_thread` * :ref:`MPI_Finalize` * :ref:`MPI_Finalized` + * :ref:`MPI_Session_init` + * :ref:`MPI_Session_finalize` diff --git a/docs/man-openmpi/man3/MPI_Session_finalize.3.rst b/docs/man-openmpi/man3/MPI_Session_finalize.3.rst index b91e2085a6b..4cf4820c1ab 100644 --- a/docs/man-openmpi/man3/MPI_Session_finalize.3.rst +++ b/docs/man-openmpi/man3/MPI_Session_finalize.3.rst @@ -56,9 +56,15 @@ DESCRIPTION :ref:`MPI_Session_finalize` releases all MPI state associated with the supplied session. Every instantiated session must be finalized using -:ref:`MPI_Session_finalize`. The handle session is set to MPI_SESSION_NULL by +:ref:`MPI_Session_finalize`. The handle session is set to ``MPI_SESSION_NULL`` by the call. +Multiple sessions can be created and destroyed during the lifetime of +an MPI process. This is different than MPI world model, which can be +initialized at most exactly once (and then subsequently finalized) +during the lifetime of an MPI process. + + NOTES ----- @@ -83,4 +89,10 @@ ERRORS .. include:: ./ERRORS.rst -.. seealso:: :ref:`MPI_Session_init` +.. seealso:: + * :ref:`MPI_Init` + * :ref:`MPI_Initialized` + * :ref:`MPI_Init_thread` + * :ref:`MPI_Finalize` + * :ref:`MPI_Finalized` + * :ref:`MPI_Session_init` diff --git a/docs/man-openmpi/man3/MPI_Session_init.3.rst b/docs/man-openmpi/man3/MPI_Session_init.3.rst index 813ad196a83..43432c3e8d6 100644 --- a/docs/man-openmpi/man3/MPI_Session_init.3.rst +++ b/docs/man-openmpi/man3/MPI_Session_init.3.rst @@ -59,19 +59,27 @@ OUTPUT PARAMETERS DESCRIPTION ----------- -:ref:`MPI_Session_init` is used to instantiate an MPI Session. The returned -session handle can be used to query the runtime system about -characteristics of the job within which the process is running, as well -as other system resources. An application can make multiple calls to -:ref:`MPI_Session_init` and the related :ref:`MPI_Session_finalize` routine. +:ref:`MPI_Session_init` is used to instantiate an MPI Session. The +returned session handle can be used to query the runtime system about +characteristics of the job within which the process is running, as +well as other system resources. Other MPI communications can also be +initiated in the context of an MPI session handle. All sessions must +be finalized via :ref:`MPI_Session_finalize` before the MPI process +terminates. + +Multiple sessions can be created and destroyed during the lifetime of +an MPI process. This is different than MPI world model, which can be +initialized at most exactly once (and then subsequently finalized) +during the lifetime of an MPI process. + NOTES ----- -The info argument is used to request MPI functionality requirements and -possible MPI implementation specific capabilities. +The *info* argument is used to request MPI functionality requirements +and possible MPI implementation specific capabilities. -The errhandler argument specifies an error handler to invoke in the +The *errhandler* argument specifies an error handler to invoke in the event that the Session instantiation call encounters an error. ERRORS @@ -79,4 +87,24 @@ ERRORS .. include:: ./ERRORS.rst -.. seealso:: :ref:`MPI_Session_get_num_psets` MPI_Session_group_from_pset +.. seealso:: + * :ref:`MPI_Init` + * :ref:`MPI_Initialized` + * :ref:`MPI_Init_thread` + * :ref:`MPI_Finalize` + * :ref:`MPI_Finalized` + * :ref:`MPI_Group_from_session_pset` + * :ref:`MPI_Session_c2f` + * :ref:`MPI_Session_call_errhandler` + * :ref:`MPI_Session_create_errhandler` + * :ref:`MPI_Session_f2c` + * :ref:`MPI_Session_finalize` + * :ref:`MPI_Session_get_errhandler` + * :ref:`MPI_Session_get_info` + * :ref:`MPI_Session_get_nth_pset` + * :ref:`MPI_Session_get_num_psets` + * :ref:`MPI_Session_get_pset_info` + * :ref:`MPI_Session_init` + * :ref:`MPI_Session_set_errhandler` + * :ref:`MPI_T_pvar_session_create` + * :ref:`MPI_T_pvar_session_free` From 1bea4f7e7e087faed20652e136eaacf6a03b6703 Mon Sep 17 00:00:00 2001 From: Aurelien Bouteiller Date: Thu, 10 Jul 2025 16:28:28 -0400 Subject: [PATCH 4/5] Add missing file MPI_Session_c2f.3.rst referenced from MPI_Session_init doc. Signed-off-by: Aurelien Bouteiller --- docs/man-openmpi/man3/MPI_Session_c2f.3.rst | 9 +++++++++ docs/man-openmpi/man3/index.rst | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/man-openmpi/man3/MPI_Session_c2f.3.rst diff --git a/docs/man-openmpi/man3/MPI_Session_c2f.3.rst b/docs/man-openmpi/man3/MPI_Session_c2f.3.rst new file mode 100644 index 00000000000..c798c3f0a7a --- /dev/null +++ b/docs/man-openmpi/man3/MPI_Session_c2f.3.rst @@ -0,0 +1,9 @@ +.. _mpi_session_c2f: + +MPI_Session_c2f +=============== + .. include_body + +.. include:: ../man3/MPI_Session_f2c.3.rst + :start-after: .. include_body + diff --git a/docs/man-openmpi/man3/index.rst b/docs/man-openmpi/man3/index.rst index 7df08ccb542..cb9961970ca 100644 --- a/docs/man-openmpi/man3/index.rst +++ b/docs/man-openmpi/man3/index.rst @@ -324,6 +324,7 @@ MPI API manual pages (section 3) MPI_Sendrecv_replace.3.rst MPI_Session_call_errhandler.3.rst MPI_Session_create_errhandler.3.rst + MPI_Session_c2f.3.rst MPI_Session_f2c.3.rst MPI_Session_finalize.3.rst MPI_Session_get_errhandler.3.rst From 91a8d34fa3c6f365ffa8e698343885bb458d9f8e Mon Sep 17 00:00:00 2001 From: Aurelien Bouteiller Date: Thu, 10 Jul 2025 17:06:20 -0400 Subject: [PATCH 5/5] Update hardcoded version values in documentation for Init_threads Signed-off-by: Aurelien Bouteiller --- docs/man-openmpi/man3/MPI_Init_thread.3.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/man-openmpi/man3/MPI_Init_thread.3.rst b/docs/man-openmpi/man3/MPI_Init_thread.3.rst index aca6307a347..dd37b866ef1 100644 --- a/docs/man-openmpi/man3/MPI_Init_thread.3.rst +++ b/docs/man-openmpi/man3/MPI_Init_thread.3.rst @@ -173,14 +173,12 @@ of the values listed below. * - ``MPI_THREAD_MULTIPLE`` - ``MPI_THREAD_MULTIPLE``, ``MULTIPLE``, 3 -.. note:: Prior to Open MPI v6.0.0, only the integer values 0 through +.. note:: In Open MPI v5.0.8 and prior, only the integer values 0 through 3 were acceptable values for the ``OMPI_MPI_THREAD_LEVEL`` environment variable. - Starting with Open MPI v6.0.0, the Open MPI community - recomends using one of the string name variants so that it - can be correctly mapped to the corresponding Open MPI ABI - value or the MPI Standard ABI value, as relevant. + Starting with Open MPI v5.0.9, the Open MPI community + recomends using one of the string name variants. NOTES -----