From 282ae1c094d5d585f05b9da0a4406cfbc135576f Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Wed, 11 Dec 2024 12:07:48 -0800 Subject: [PATCH 01/18] added non-blocking root communicator --- src/axom/lumberjack/MPIUtility.cpp | 37 ++++- src/axom/lumberjack/MPIUtility.hpp | 15 +- .../NonBlockingRootCommunicator.cpp | 91 ++++++++++++ .../NonBlockingRootCommunicator.hpp | 135 ++++++++++++++++++ 4 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 src/axom/lumberjack/NonBlockingRootCommunicator.cpp create mode 100644 src/axom/lumberjack/NonBlockingRootCommunicator.hpp diff --git a/src/axom/lumberjack/MPIUtility.cpp b/src/axom/lumberjack/MPIUtility.cpp index 34a4c96b5e..a12f4befde 100644 --- a/src/axom/lumberjack/MPIUtility.cpp +++ b/src/axom/lumberjack/MPIUtility.cpp @@ -49,16 +49,49 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm) return charArray; } +const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag) +{ + const int mpiTag = (tag == false) ? LJ_TAG : tag; + char* charArray = nullptr; + int messageSize = -1; + MPI_Status mpiStatus; + + // Get size and source of MPI message + int mpiFlag = true; + MPI_Iprobe(MPI_ANY_SOURCE, tag, comm, &mpiFlag, &mpiStatus); + + if (mpiFlag == true) { + MPI_Get_count(&mpiStatus, MPI_CHAR, &messageSize); + + // Setup where to receive the char array + charArray = new char[messageSize + 1]; + charArray[messageSize] = '\0'; + + // Receive packed Message + MPI_Recv(charArray, + messageSize, + MPI_CHAR, + mpiStatus.MPI_SOURCE, + mpiTag, + comm, + &mpiStatus); + } + + return charArray; +} + void mpiNonBlockingSendMessages(MPI_Comm comm, int destinationRank, - const char* packedMessagesToBeSent) + const char* packedMessagesToBeSent, + const int tag) { + const int mpiTag = (tag == false) ? LJ_TAG : tag; MPI_Request mpiRequest; MPI_Isend(const_cast(packedMessagesToBeSent), strlen(packedMessagesToBeSent), MPI_CHAR, destinationRank, - LJ_TAG, + mpiTag, comm, &mpiRequest); MPI_Request_free(&mpiRequest); diff --git a/src/axom/lumberjack/MPIUtility.hpp b/src/axom/lumberjack/MPIUtility.hpp index 1d499ea839..1842eada49 100644 --- a/src/axom/lumberjack/MPIUtility.hpp +++ b/src/axom/lumberjack/MPIUtility.hpp @@ -30,6 +30,16 @@ namespace lumberjack */ const char* mpiBlockingReceiveMessages(MPI_Comm comm); +/*! + ***************************************************************************** + * \brief Receives any Message sent to this rank, if there are any messages + * that are sent. Returns null if no messages are sent. + * + * \param [in] comm The MPI Communicator. + ***************************************************************************** + */ +const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag = 0); + /*! ***************************************************************************** * \brief Sends all Message sent to the given rank. @@ -40,11 +50,14 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm); * \param [in] destinationRank Where the Message classes is being sent. * \param [in,out] packedMessagesToBeSent All of the Message classes to be sent * packed together. + * \param [in] tag The MPI tag to use for communication. When set to zero, + * MPI communication uses default LJ_Tag. ***************************************************************************** */ void mpiNonBlockingSendMessages(MPI_Comm comm, int destinationRank, - const char* packedMessagesToBeSent); + const char* packedMessagesToBeSent, + const int tag = 0); } // end namespace lumberjack } // end namespace axom diff --git a/src/axom/lumberjack/NonBlockingRootCommunicator.cpp b/src/axom/lumberjack/NonBlockingRootCommunicator.cpp new file mode 100644 index 0000000000..a1bc6040a9 --- /dev/null +++ b/src/axom/lumberjack/NonBlockingRootCommunicator.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and +// other Axom Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/*! + ****************************************************************************** + * + * \file NonBlockingRootCommunicator.cpp + * + * \brief Implementation of the NonBlockingRootCommunicator class. + * + ****************************************************************************** + */ + +#include "axom/lumberjack/NonBlockingRootCommunicator.hpp" +#include "axom/lumberjack/MPIUtility.hpp" + +namespace axom +{ +namespace lumberjack +{ +void NonBlockingRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) +{ + m_mpiComm = comm; + MPI_Comm_rank(m_mpiComm, &m_mpiCommRank); + MPI_Comm_size(m_mpiComm, &m_mpiCommSize); + m_ranksLimit = ranksLimit; +} + +void NonBlockingRootCommunicator::finalize() { } + +int NonBlockingRootCommunicator::rank() { return m_mpiCommRank; } + +void NonBlockingRootCommunicator::ranksLimit(int value) { m_ranksLimit = value; } + +int NonBlockingRootCommunicator::ranksLimit() { return m_ranksLimit; } + +int NonBlockingRootCommunicator::numPushesToFlush() { return 1; } + +void NonBlockingRootCommunicator::push(const char* packedMessagesToBeSent, + std::vector& receivedPackedMessages) +{ + constexpr int mpiTag = 32767; + if(m_mpiCommRank == 0) + { + const char* currPackedMessages = nullptr; + bool receive_messages = true; + while(receive_messages) + { + currPackedMessages = mpiNonBlockingReceiveMessages(m_mpiComm, mpiTag); + + if(isPackedMessagesEmpty(currPackedMessages)) + { + if (currPackedMessages == nullptr ) + { + receive_messages = false; + } else { + delete [] currPackedMessages; + } + + } + else + { + receivedPackedMessages.push_back(currPackedMessages); + } + + currPackedMessages = nullptr; + + } + } + else + { + if(isPackedMessagesEmpty(packedMessagesToBeSent) == false) + { + mpiNonBlockingSendMessages(m_mpiComm, 0, packedMessagesToBeSent, mpiTag); + } + } +} + +bool NonBlockingRootCommunicator::isOutputNode() +{ + if(m_mpiCommRank == 0) + { + return true; + } + return false; +} + +} // end namespace lumberjack +} // end namespace axom \ No newline at end of file diff --git a/src/axom/lumberjack/NonBlockingRootCommunicator.hpp b/src/axom/lumberjack/NonBlockingRootCommunicator.hpp new file mode 100644 index 0000000000..ec432ef22c --- /dev/null +++ b/src/axom/lumberjack/NonBlockingRootCommunicator.hpp @@ -0,0 +1,135 @@ +// Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and +// other Axom Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/*! + ******************************************************************************* + * \file NonBlockingRootCommunicator.hpp + * + * \brief This file contains the class definition of the + * NonBlockingRootCommunicator. + ******************************************************************************* + */ + +#ifndef NONBLOCKINGROOTCOMMUNICATOR_HPP +#define NONBLOCKINGROOTCOMMUNICATOR_HPP + +#include "axom/lumberjack/Lumberjack.hpp" +#include "axom/lumberjack/Communicator.hpp" + +namespace axom +{ +namespace lumberjack +{ +/*! + ******************************************************************************* + * \class NonBlockingRootCommunicator + * + * \brief Based off of RootCommunicator. This communicator pushes + messages from any rank to root non-collectively, if any messages are sent. + ******************************************************************************* + */ +class NonBlockingRootCommunicator : public axom::lumberjack::Communicator +{ +public: + /*! + ***************************************************************************** + * \brief Called to initialize the Communicator. + * + * This performs any setup work the Communicator needs before doing any work. + * It is required that this is called before using the Communicator. + * + * \param [in] comm The MPI Communicator + * \param [in] ranksLimit Limit on how many ranks are individually tracked per + * Message. + ***************************************************************************** + */ + void initialize(MPI_Comm comm, int ranksLimit); + + /*! + ***************************************************************************** + * \brief Called to finalize the Communicator. + * + * This performs any cleanup work the Communicator needs to do before going + * away.It is required that this is the last function called by the + * Communicator. + ***************************************************************************** + */ + void finalize(); + + /*! + ***************************************************************************** + * \brief Returns the MPI rank of this node + ***************************************************************************** + */ + int rank(); + + /*! + ***************************************************************************** + * \brief Sets the rank limit. + * + * This is the limit on how many ranks generated a given message are + * individually tracked per Message. After the limit has been reached, only + * the Message::rankCount is incremented. + * + * \param [in] value Limits how many ranks are tracked per Message. + ***************************************************************************** + */ + void ranksLimit(int value); + + /*! + ***************************************************************************** + * \brief Returns the rank limit. + * + * This is the limit on how many ranks generated a given message are + * individually tracked per Message. After the limit has been reached, only + * the Message::rankCount is incremented. + ***************************************************************************** + */ + int ranksLimit(); + + /*! + ***************************************************************************** + * \brief Function used by the Lumberjack class to indicate how many + * individual pushes fully flush all currently held Message classes to the + * root node. The Communicator class's tree structure dictates this. + ***************************************************************************** + */ + int numPushesToFlush(); + + /*! + ***************************************************************************** + * \brief This pushes all messages to the root node. + * + * All messages are pushed to the root node. This is the same as + * RootCommunicator::pushMessagesFully for this Communicator. + * + * \param [in] packedMessagesToBeSent All of this rank's Message classes + * packed into a single buffer. + * \param [in,out] receivedPackedMessages Received packed message buffers from + * this nodes children. + ***************************************************************************** + */ + void push(const char* packedMessagesToBeSent, + std::vector& receivedPackedMessages); + + /*! + ***************************************************************************** + * \brief Function used by the Lumberjack to indicate whether this node should + * be outputting messages. Only the root node outputs messages. + ***************************************************************************** + */ + bool isOutputNode(); + +private: + MPI_Comm m_mpiComm; + int m_mpiCommRank; + int m_mpiCommSize; + int m_ranksLimit; +}; + +} // end namespace lumberjack +} // end namespace axom + +#endif \ No newline at end of file From 43b02dada87e0d7321832273a1b720e805106ed3 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 12 Dec 2024 17:10:56 -0800 Subject: [PATCH 02/18] fixed tag and flag comparison type, renamed NonBlockingRootCommunicator to NonCollectiveRootCommunicator --- src/axom/lumberjack/MPIUtility.cpp | 12 +++++----- src/axom/lumberjack/MPIUtility.hpp | 4 ++-- ....cpp => NonCollectiveRootCommunicator.cpp} | 22 +++++++++---------- ....hpp => NonCollectiveRootCommunicator.hpp} | 12 +++++----- 4 files changed, 25 insertions(+), 25 deletions(-) rename src/axom/lumberjack/{NonBlockingRootCommunicator.cpp => NonCollectiveRootCommunicator.cpp} (68%) rename src/axom/lumberjack/{NonBlockingRootCommunicator.hpp => NonCollectiveRootCommunicator.hpp} (94%) diff --git a/src/axom/lumberjack/MPIUtility.cpp b/src/axom/lumberjack/MPIUtility.cpp index a12f4befde..0859da8ea8 100644 --- a/src/axom/lumberjack/MPIUtility.cpp +++ b/src/axom/lumberjack/MPIUtility.cpp @@ -49,18 +49,18 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm) return charArray; } -const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag) +const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag) { - const int mpiTag = (tag == false) ? LJ_TAG : tag; + const int mpiTag = (tag == 0) ? LJ_TAG : tag; char* charArray = nullptr; int messageSize = -1; MPI_Status mpiStatus; // Get size and source of MPI message - int mpiFlag = true; + int mpiFlag = 0; MPI_Iprobe(MPI_ANY_SOURCE, tag, comm, &mpiFlag, &mpiStatus); - if (mpiFlag == true) { + if (mpiFlag == 1) { MPI_Get_count(&mpiStatus, MPI_CHAR, &messageSize); // Setup where to receive the char array @@ -83,9 +83,9 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag) void mpiNonBlockingSendMessages(MPI_Comm comm, int destinationRank, const char* packedMessagesToBeSent, - const int tag) + int tag) { - const int mpiTag = (tag == false) ? LJ_TAG : tag; + const int mpiTag = (tag == 0) ? LJ_TAG : tag; MPI_Request mpiRequest; MPI_Isend(const_cast(packedMessagesToBeSent), strlen(packedMessagesToBeSent), diff --git a/src/axom/lumberjack/MPIUtility.hpp b/src/axom/lumberjack/MPIUtility.hpp index 1842eada49..0fcdd0a926 100644 --- a/src/axom/lumberjack/MPIUtility.hpp +++ b/src/axom/lumberjack/MPIUtility.hpp @@ -38,7 +38,7 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm); * \param [in] comm The MPI Communicator. ***************************************************************************** */ -const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag = 0); +const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag = 0); /*! ***************************************************************************** @@ -57,7 +57,7 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, const int tag = 0); void mpiNonBlockingSendMessages(MPI_Comm comm, int destinationRank, const char* packedMessagesToBeSent, - const int tag = 0); + int tag = 0); } // end namespace lumberjack } // end namespace axom diff --git a/src/axom/lumberjack/NonBlockingRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp similarity index 68% rename from src/axom/lumberjack/NonBlockingRootCommunicator.cpp rename to src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index a1bc6040a9..d2076a8059 100644 --- a/src/axom/lumberjack/NonBlockingRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -6,21 +6,21 @@ /*! ****************************************************************************** * - * \file NonBlockingRootCommunicator.cpp + * \file NonCollectiveRootCommunicator.cpp * - * \brief Implementation of the NonBlockingRootCommunicator class. + * \brief Implementation of the NonCollectiveRootCommunicator class. * ****************************************************************************** */ -#include "axom/lumberjack/NonBlockingRootCommunicator.hpp" +#include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" #include "axom/lumberjack/MPIUtility.hpp" namespace axom { namespace lumberjack { -void NonBlockingRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) +void NonCollectiveRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) { m_mpiComm = comm; MPI_Comm_rank(m_mpiComm, &m_mpiCommRank); @@ -28,17 +28,17 @@ void NonBlockingRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) m_ranksLimit = ranksLimit; } -void NonBlockingRootCommunicator::finalize() { } +void NonCollectiveRootCommunicator::finalize() { } -int NonBlockingRootCommunicator::rank() { return m_mpiCommRank; } +int NonCollectiveRootCommunicator::rank() { return m_mpiCommRank; } -void NonBlockingRootCommunicator::ranksLimit(int value) { m_ranksLimit = value; } +void NonCollectiveRootCommunicator::ranksLimit(int value) { m_ranksLimit = value; } -int NonBlockingRootCommunicator::ranksLimit() { return m_ranksLimit; } +int NonCollectiveRootCommunicator::ranksLimit() { return m_ranksLimit; } -int NonBlockingRootCommunicator::numPushesToFlush() { return 1; } +int NonCollectiveRootCommunicator::numPushesToFlush() { return 1; } -void NonBlockingRootCommunicator::push(const char* packedMessagesToBeSent, +void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, std::vector& receivedPackedMessages) { constexpr int mpiTag = 32767; @@ -78,7 +78,7 @@ void NonBlockingRootCommunicator::push(const char* packedMessagesToBeSent, } } -bool NonBlockingRootCommunicator::isOutputNode() +bool NonCollectiveRootCommunicator::isOutputNode() { if(m_mpiCommRank == 0) { diff --git a/src/axom/lumberjack/NonBlockingRootCommunicator.hpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp similarity index 94% rename from src/axom/lumberjack/NonBlockingRootCommunicator.hpp rename to src/axom/lumberjack/NonCollectiveRootCommunicator.hpp index ec432ef22c..fb2ff23c16 100644 --- a/src/axom/lumberjack/NonBlockingRootCommunicator.hpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp @@ -5,15 +5,15 @@ /*! ******************************************************************************* - * \file NonBlockingRootCommunicator.hpp + * \file NonCollectiveRootCommunicator.hpp * * \brief This file contains the class definition of the - * NonBlockingRootCommunicator. + * NonCollectiveRootCommunicator. ******************************************************************************* */ -#ifndef NONBLOCKINGROOTCOMMUNICATOR_HPP -#define NONBLOCKINGROOTCOMMUNICATOR_HPP +#ifndef NONCOLLECTIVEROOTCOMMUNICATOR_HPP +#define NONCOLLECTIVEROOTCOMMUNICATOR_HPP #include "axom/lumberjack/Lumberjack.hpp" #include "axom/lumberjack/Communicator.hpp" @@ -24,13 +24,13 @@ namespace lumberjack { /*! ******************************************************************************* - * \class NonBlockingRootCommunicator + * \class NonCollectiveRootCommunicator * * \brief Based off of RootCommunicator. This communicator pushes messages from any rank to root non-collectively, if any messages are sent. ******************************************************************************* */ -class NonBlockingRootCommunicator : public axom::lumberjack::Communicator +class NonCollectiveRootCommunicator : public axom::lumberjack::Communicator { public: /*! From 7969aa8577f322a1bc24bba351173cd2b96643fa Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 12 Dec 2024 17:15:01 -0800 Subject: [PATCH 03/18] small formatting fix --- src/axom/lumberjack/NonCollectiveRootCommunicator.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index d2076a8059..879b9738bb 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -55,7 +55,9 @@ void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, if (currPackedMessages == nullptr ) { receive_messages = false; - } else { + } + else + { delete [] currPackedMessages; } From 14f802e4dac9a6200fe6d87560f171dc7b4e4146 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Fri, 13 Dec 2024 15:53:51 -0800 Subject: [PATCH 04/18] added unique mpi tag for each NonCollectiveRootCommunicator instance, and added unit testing --- src/axom/lumberjack/CMakeLists.txt | 2 + .../NonCollectiveRootCommunicator.cpp | 10 +- .../NonCollectiveRootCommunicator.hpp | 8 ++ src/axom/lumberjack/tests/CMakeLists.txt | 3 +- ...mberjack_NonCollectiveRootCommunicator.hpp | 99 +++++++++++++++++++ .../lumberjack/tests/lumberjack_mpi_main.cpp | 1 + 6 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp diff --git a/src/axom/lumberjack/CMakeLists.txt b/src/axom/lumberjack/CMakeLists.txt index a685825fd7..27de5adcd9 100644 --- a/src/axom/lumberjack/CMakeLists.txt +++ b/src/axom/lumberjack/CMakeLists.txt @@ -17,6 +17,7 @@ set(lumberjack_headers Message.hpp MPIUtility.hpp RootCommunicator.hpp + NonCollectiveRootCommunicator.hpp TextEqualityCombiner.hpp TextTagCombiner.hpp ) @@ -27,6 +28,7 @@ set(lumberjack_sources Message.cpp MPIUtility.cpp RootCommunicator.cpp + NonCollectiveRootCommunicator.cpp ) diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index 879b9738bb..3cef663370 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -22,10 +22,13 @@ namespace lumberjack { void NonCollectiveRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) { + static int mpiTag = 32767; m_mpiComm = comm; MPI_Comm_rank(m_mpiComm, &m_mpiCommRank); MPI_Comm_size(m_mpiComm, &m_mpiCommSize); m_ranksLimit = ranksLimit; + m_mpiTag = mpiTag; + ++mpiTag; } void NonCollectiveRootCommunicator::finalize() { } @@ -41,14 +44,13 @@ int NonCollectiveRootCommunicator::numPushesToFlush() { return 1; } void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, std::vector& receivedPackedMessages) { - constexpr int mpiTag = 32767; if(m_mpiCommRank == 0) { const char* currPackedMessages = nullptr; bool receive_messages = true; while(receive_messages) { - currPackedMessages = mpiNonBlockingReceiveMessages(m_mpiComm, mpiTag); + currPackedMessages = mpiNonBlockingReceiveMessages(m_mpiComm, m_mpiTag); if(isPackedMessagesEmpty(currPackedMessages)) { @@ -75,7 +77,7 @@ void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, { if(isPackedMessagesEmpty(packedMessagesToBeSent) == false) { - mpiNonBlockingSendMessages(m_mpiComm, 0, packedMessagesToBeSent, mpiTag); + mpiNonBlockingSendMessages(m_mpiComm, 0, packedMessagesToBeSent, m_mpiTag); } } } @@ -89,5 +91,7 @@ bool NonCollectiveRootCommunicator::isOutputNode() return false; } +int NonCollectiveRootCommunicator::mpiTag() const { return m_mpiTag; } + } // end namespace lumberjack } // end namespace axom \ No newline at end of file diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp index fb2ff23c16..946ade76f2 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp @@ -122,11 +122,19 @@ class NonCollectiveRootCommunicator : public axom::lumberjack::Communicator */ bool isOutputNode(); + /*! + ***************************************************************************** + * \brief Returns the MPI tag used for this communicator. + ***************************************************************************** + */ + int mpiTag() const; + private: MPI_Comm m_mpiComm; int m_mpiCommRank; int m_mpiCommSize; int m_ranksLimit; + int m_mpiTag; }; } // end namespace lumberjack diff --git a/src/axom/lumberjack/tests/CMakeLists.txt b/src/axom/lumberjack/tests/CMakeLists.txt index 7d95e8aff3..9d8dc2a16b 100644 --- a/src/axom/lumberjack/tests/CMakeLists.txt +++ b/src/axom/lumberjack/tests/CMakeLists.txt @@ -34,7 +34,8 @@ endforeach() #------------------------------------------------------------------------------ set(lumberjack_mpi_tests lumberjack_BinaryCommunicator.hpp - lumberjack_RootCommunicator.hpp ) + lumberjack_RootCommunicator.hpp + lumberjack_NonCollectiveRootCommunicator.hpp ) axom_add_executable(NAME lumberjack_mpi_tests SOURCES lumberjack_mpi_main.cpp diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp new file mode 100644 index 0000000000..fd3f3bf2c4 --- /dev/null +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -0,0 +1,99 @@ +// Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and +// other Axom Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +#include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" +#include "gtest/gtest.h" +#include "mpi.h" + +TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) +{ + MPI_Barrier(MPI_COMM_WORLD); + + int commSize = -1; + MPI_Comm_size(MPI_COMM_WORLD, &commSize); + + const int ranksLimit = 5; + axom::lumberjack::NonCollectiveRootCommunicator c; + + c.initialize(MPI_COMM_WORLD, ranksLimit); + + std::string message = std::to_string(c.rank()); + + std::vector receivedPackedMessages; + + // send message only from even ranks + if ((c.rank() % 2) == 0) + { + c.push(message.c_str(), receivedPackedMessages); + } + + if(c.rank() != 0) + { + EXPECT_EQ((int)receivedPackedMessages.size(), 0); + } + else + { + const int numMessagesToReceive = ((commSize % 2) == 0) ? ((commSize / 2) - 1) : (commSize / 2); + EXPECT_EQ((int)receivedPackedMessages.size(), numMessagesToReceive); + for(int i = 1; i <= numMessagesToReceive; ++i) + { + std::string currMessage = std::to_string(i*2); + bool found = false; + for(auto& rm : receivedPackedMessages) + { + if(strcmp(rm, currMessage.c_str()) == 0) + { + found = true; + } + } + if(!found) + { + std::cout << "Error: Message not received:" << currMessage << std::endl; + } + EXPECT_EQ(found, true); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + c.finalize(); + + MPI_Barrier(MPI_COMM_WORLD); + + // cleanup allocated memory from received messages + for(auto& rm : receivedPackedMessages) + { + delete rm; + } + + receivedPackedMessages.clear(); + +} + +TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) +{ + MPI_Barrier(MPI_COMM_WORLD); + + int commSize = -1; + MPI_Comm_size(MPI_COMM_WORLD, &commSize); + + const int ranksLimit = 5; + axom::lumberjack::NonCollectiveRootCommunicator c1; + axom::lumberjack::NonCollectiveRootCommunicator c2; + + c1.initialize(MPI_COMM_WORLD, ranksLimit); + c2.initialize(MPI_COMM_WORLD, ranksLimit); + + // each communicator has unique MPI tag + EXPECT_NE(c1.mpiTag(), c2.mpiTag()); + + MPI_Barrier(MPI_COMM_WORLD); + + c1.finalize(); + c2.finalize(); + + MPI_Barrier(MPI_COMM_WORLD); + +} \ No newline at end of file diff --git a/src/axom/lumberjack/tests/lumberjack_mpi_main.cpp b/src/axom/lumberjack/tests/lumberjack_mpi_main.cpp index 616f7e45af..e990cc9233 100644 --- a/src/axom/lumberjack/tests/lumberjack_mpi_main.cpp +++ b/src/axom/lumberjack/tests/lumberjack_mpi_main.cpp @@ -9,6 +9,7 @@ #include "lumberjack_BinaryCommunicator.hpp" #include "lumberjack_RootCommunicator.hpp" +#include "lumberjack_NonCollectiveRootCommunicator.hpp" int main(int argc, char* argv[]) { From c04745aa5b98885d96ef54492142a5f852088d38 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Mon, 16 Dec 2024 10:19:40 -0800 Subject: [PATCH 05/18] added missing function param description --- src/axom/lumberjack/MPIUtility.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/axom/lumberjack/MPIUtility.hpp b/src/axom/lumberjack/MPIUtility.hpp index 0fcdd0a926..701087e668 100644 --- a/src/axom/lumberjack/MPIUtility.hpp +++ b/src/axom/lumberjack/MPIUtility.hpp @@ -36,6 +36,8 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm); * that are sent. Returns null if no messages are sent. * * \param [in] comm The MPI Communicator. + * \param [in] tag The MPI tag to use for communication. When set to zero, + * MPI communication uses default LJ_Tag. ***************************************************************************** */ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag = 0); From 8275b6d393fa70096f4aa34ddfe5fd1c0a7f5d66 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 16 Jan 2025 11:04:53 -0800 Subject: [PATCH 06/18] applied clang-format --- src/axom/lumberjack/MPIUtility.cpp | 15 ++++++++------- .../lumberjack/NonCollectiveRootCommunicator.cpp | 16 +++++++++------- .../lumberjack_NonCollectiveRootCommunicator.hpp | 13 ++++++------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/axom/lumberjack/MPIUtility.cpp b/src/axom/lumberjack/MPIUtility.cpp index 0859da8ea8..6f7063b205 100644 --- a/src/axom/lumberjack/MPIUtility.cpp +++ b/src/axom/lumberjack/MPIUtility.cpp @@ -60,7 +60,8 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag) int mpiFlag = 0; MPI_Iprobe(MPI_ANY_SOURCE, tag, comm, &mpiFlag, &mpiStatus); - if (mpiFlag == 1) { + if(mpiFlag == 1) + { MPI_Get_count(&mpiStatus, MPI_CHAR, &messageSize); // Setup where to receive the char array @@ -69,12 +70,12 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag) // Receive packed Message MPI_Recv(charArray, - messageSize, - MPI_CHAR, - mpiStatus.MPI_SOURCE, - mpiTag, - comm, - &mpiStatus); + messageSize, + MPI_CHAR, + mpiStatus.MPI_SOURCE, + mpiTag, + comm, + &mpiStatus); } return charArray; diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index 3cef663370..37f374e495 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -35,14 +35,18 @@ void NonCollectiveRootCommunicator::finalize() { } int NonCollectiveRootCommunicator::rank() { return m_mpiCommRank; } -void NonCollectiveRootCommunicator::ranksLimit(int value) { m_ranksLimit = value; } +void NonCollectiveRootCommunicator::ranksLimit(int value) +{ + m_ranksLimit = value; +} int NonCollectiveRootCommunicator::ranksLimit() { return m_ranksLimit; } int NonCollectiveRootCommunicator::numPushesToFlush() { return 1; } -void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, - std::vector& receivedPackedMessages) +void NonCollectiveRootCommunicator::push( + const char* packedMessagesToBeSent, + std::vector& receivedPackedMessages) { if(m_mpiCommRank == 0) { @@ -54,15 +58,14 @@ void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, if(isPackedMessagesEmpty(currPackedMessages)) { - if (currPackedMessages == nullptr ) + if(currPackedMessages == nullptr) { receive_messages = false; } else { - delete [] currPackedMessages; + delete[] currPackedMessages; } - } else { @@ -70,7 +73,6 @@ void NonCollectiveRootCommunicator::push(const char* packedMessagesToBeSent, } currPackedMessages = nullptr; - } } else diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index fd3f3bf2c4..52f4813862 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -13,7 +13,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) int commSize = -1; MPI_Comm_size(MPI_COMM_WORLD, &commSize); - + const int ranksLimit = 5; axom::lumberjack::NonCollectiveRootCommunicator c; @@ -24,7 +24,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) std::vector receivedPackedMessages; // send message only from even ranks - if ((c.rank() % 2) == 0) + if((c.rank() % 2) == 0) { c.push(message.c_str(), receivedPackedMessages); } @@ -35,11 +35,12 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) } else { - const int numMessagesToReceive = ((commSize % 2) == 0) ? ((commSize / 2) - 1) : (commSize / 2); + const int numMessagesToReceive = + ((commSize % 2) == 0) ? ((commSize / 2) - 1) : (commSize / 2); EXPECT_EQ((int)receivedPackedMessages.size(), numMessagesToReceive); for(int i = 1; i <= numMessagesToReceive; ++i) { - std::string currMessage = std::to_string(i*2); + std::string currMessage = std::to_string(i * 2); bool found = false; for(auto& rm : receivedPackedMessages) { @@ -69,7 +70,6 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) } receivedPackedMessages.clear(); - } TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) @@ -78,7 +78,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) int commSize = -1; MPI_Comm_size(MPI_COMM_WORLD, &commSize); - + const int ranksLimit = 5; axom::lumberjack::NonCollectiveRootCommunicator c1; axom::lumberjack::NonCollectiveRootCommunicator c2; @@ -95,5 +95,4 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) c2.finalize(); MPI_Barrier(MPI_COMM_WORLD); - } \ No newline at end of file From 0c0da25dcdb097cbebc7f9d08c8eef6b62ce1dc0 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 16 Jan 2025 15:00:42 -0800 Subject: [PATCH 07/18] fixed noncollective_communication test --- src/axom/lumberjack/MPIUtility.cpp | 1 - .../lumberjack/NonCollectiveRootCommunicator.cpp | 2 +- .../lumberjack_NonCollectiveRootCommunicator.hpp | 16 ++++++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/axom/lumberjack/MPIUtility.cpp b/src/axom/lumberjack/MPIUtility.cpp index 6f7063b205..161adaa032 100644 --- a/src/axom/lumberjack/MPIUtility.cpp +++ b/src/axom/lumberjack/MPIUtility.cpp @@ -14,7 +14,6 @@ */ #include "axom/lumberjack/MPIUtility.hpp" - #include namespace axom diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index 37f374e495..f74d9cadef 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -96,4 +96,4 @@ bool NonCollectiveRootCommunicator::isOutputNode() int NonCollectiveRootCommunicator::mpiTag() const { return m_mpiTag; } } // end namespace lumberjack -} // end namespace axom \ No newline at end of file +} // end namespace axom diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index 52f4813862..8a08672f8c 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -23,8 +23,16 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) std::vector receivedPackedMessages; - // send message only from even ranks - if((c.rank() % 2) == 0) + // send message only from even ranks that are non-zero + if((c.rank() % 2) == 0 && c.rank() != 0) + { + c.push(message.c_str(), receivedPackedMessages); + } + + MPI_Barrier(MPI_COMM_WORLD); + + // receive messages from rank 0 after barrier + if(c.rank() == 0) { c.push(message.c_str(), receivedPackedMessages); } @@ -51,7 +59,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) } if(!found) { - std::cout << "Error: Message not received:" << currMessage << std::endl; + std::cout << "Error: Message not received: " << currMessage << std::endl; } EXPECT_EQ(found, true); } @@ -95,4 +103,4 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) c2.finalize(); MPI_Barrier(MPI_COMM_WORLD); -} \ No newline at end of file +} From 1c120d35448785f5191ab053055a0d497205cb5e Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Wed, 22 Jan 2025 09:39:05 -0800 Subject: [PATCH 08/18] small fix for delete --- .../tests/lumberjack_NonCollectiveRootCommunicator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index 8a08672f8c..e7389cb804 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -74,7 +74,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) // cleanup allocated memory from received messages for(auto& rm : receivedPackedMessages) { - delete rm; + delete[] rm; } receivedPackedMessages.clear(); From c74d1f4924ea3688441acf2796763b2a611c0847 Mon Sep 17 00:00:00 2001 From: gberg617 <63525600+gberg617@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:18:56 -0800 Subject: [PATCH 09/18] Update src/axom/lumberjack/MPIUtility.hpp Co-authored-by: Chris White --- src/axom/lumberjack/MPIUtility.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/lumberjack/MPIUtility.hpp b/src/axom/lumberjack/MPIUtility.hpp index 701087e668..fe7e7777bb 100644 --- a/src/axom/lumberjack/MPIUtility.hpp +++ b/src/axom/lumberjack/MPIUtility.hpp @@ -53,7 +53,7 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag = 0); * \param [in,out] packedMessagesToBeSent All of the Message classes to be sent * packed together. * \param [in] tag The MPI tag to use for communication. When set to zero, - * MPI communication uses default LJ_Tag. + * MPI communication uses the default Lumberjack Tag. ***************************************************************************** */ void mpiNonBlockingSendMessages(MPI_Comm comm, From 53c0f4e8c6ebc4e2d8f9e8ea9521cd81c157ded3 Mon Sep 17 00:00:00 2001 From: gberg617 <63525600+gberg617@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:19:07 -0800 Subject: [PATCH 10/18] Update src/axom/lumberjack/NonCollectiveRootCommunicator.hpp Co-authored-by: Chris White --- src/axom/lumberjack/NonCollectiveRootCommunicator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp index 946ade76f2..3a720ce534 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp @@ -140,4 +140,4 @@ class NonCollectiveRootCommunicator : public axom::lumberjack::Communicator } // end namespace lumberjack } // end namespace axom -#endif \ No newline at end of file +#endif From 173bb414ecd7aea792410771b189e0b62329dbdc Mon Sep 17 00:00:00 2001 From: gberg617 <63525600+gberg617@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:19:17 -0800 Subject: [PATCH 11/18] Update src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp Co-authored-by: Chris White --- .../tests/lumberjack_NonCollectiveRootCommunicator.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index e7389cb804..4a1ad4e622 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -57,11 +57,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) found = true; } } - if(!found) - { - std::cout << "Error: Message not received: " << currMessage << std::endl; - } - EXPECT_EQ(found, true); + EXPECT_EQ(found, true) << "Message not received: " << currMessage << std::endl; } } From d946bf9146f2603944be74cb2d5255fb56c13608 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 23 Jan 2025 10:06:00 -0800 Subject: [PATCH 12/18] added comment and fixed some formatting --- src/axom/lumberjack/NonCollectiveRootCommunicator.cpp | 4 ++++ .../tests/lumberjack_NonCollectiveRootCommunicator.hpp | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index f74d9cadef..0ae91e6b95 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -28,6 +28,10 @@ void NonCollectiveRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) MPI_Comm_size(m_mpiComm, &m_mpiCommSize); m_ranksLimit = ranksLimit; m_mpiTag = mpiTag; + /* + Each communicator that sends/receives messages non-collectively needs its + own mpiTag in order to not interfere with other communicators. + */ ++mpiTag; } diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index 4a1ad4e622..3cedd046c1 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -57,7 +57,8 @@ TEST(lumberjack_NonCollectiveRootCommunicator, noncollective_communication) found = true; } } - EXPECT_EQ(found, true) << "Message not received: " << currMessage << std::endl; + EXPECT_EQ(found, true) + << "Message not received: " << currMessage << std::endl; } } From 3a6c50aa986024d07b8f879b8007895bcff7ffec Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 23 Jan 2025 11:43:55 -0800 Subject: [PATCH 13/18] add numeric_limits check for mpiTag --- src/axom/lumberjack/NonCollectiveRootCommunicator.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index 0ae91e6b95..5fa68d3f2b 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -13,6 +13,8 @@ ****************************************************************************** */ +#include + #include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" #include "axom/lumberjack/MPIUtility.hpp" @@ -32,7 +34,14 @@ void NonCollectiveRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) Each communicator that sends/receives messages non-collectively needs its own mpiTag in order to not interfere with other communicators. */ - ++mpiTag; + if(mpiTag == std::numeric_limits::max()) + { + mpiTag = 32767; + } + else + { + ++mpiTag; + } } void NonCollectiveRootCommunicator::finalize() { } From d3bf633e14d94e90fa7f12efeb86dee9eccf38c4 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Thu, 23 Jan 2025 12:46:47 -0800 Subject: [PATCH 14/18] re-run CI commit From ffb6d913c6343d8fed12df50deac99a4a71f0de4 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Tue, 28 Jan 2025 16:23:26 -0800 Subject: [PATCH 15/18] removed mpi tags for NonCollectiveRootCommunicator and added MPI comm duplicate --- src/axom/lumberjack/MPIUtility.cpp | 14 ++--- src/axom/lumberjack/MPIUtility.hpp | 5 +- .../NonCollectiveRootCommunicator.cpp | 26 ++------- .../NonCollectiveRootCommunicator.hpp | 8 --- ...mberjack_NonCollectiveRootCommunicator.hpp | 55 ++++++++++++++++++- 5 files changed, 66 insertions(+), 42 deletions(-) diff --git a/src/axom/lumberjack/MPIUtility.cpp b/src/axom/lumberjack/MPIUtility.cpp index 161adaa032..420a5c42a2 100644 --- a/src/axom/lumberjack/MPIUtility.cpp +++ b/src/axom/lumberjack/MPIUtility.cpp @@ -15,6 +15,7 @@ #include "axom/lumberjack/MPIUtility.hpp" #include +#include namespace axom { @@ -48,16 +49,15 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm) return charArray; } -const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag) +const char* mpiNonBlockingReceiveMessages(MPI_Comm comm) { - const int mpiTag = (tag == 0) ? LJ_TAG : tag; char* charArray = nullptr; int messageSize = -1; MPI_Status mpiStatus; // Get size and source of MPI message int mpiFlag = 0; - MPI_Iprobe(MPI_ANY_SOURCE, tag, comm, &mpiFlag, &mpiStatus); + MPI_Iprobe(MPI_ANY_SOURCE, LJ_TAG, comm, &mpiFlag, &mpiStatus); if(mpiFlag == 1) { @@ -72,7 +72,7 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag) messageSize, MPI_CHAR, mpiStatus.MPI_SOURCE, - mpiTag, + LJ_TAG, comm, &mpiStatus); } @@ -82,16 +82,14 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag) void mpiNonBlockingSendMessages(MPI_Comm comm, int destinationRank, - const char* packedMessagesToBeSent, - int tag) + const char* packedMessagesToBeSent) { - const int mpiTag = (tag == 0) ? LJ_TAG : tag; MPI_Request mpiRequest; MPI_Isend(const_cast(packedMessagesToBeSent), strlen(packedMessagesToBeSent), MPI_CHAR, destinationRank, - mpiTag, + LJ_TAG, comm, &mpiRequest); MPI_Request_free(&mpiRequest); diff --git a/src/axom/lumberjack/MPIUtility.hpp b/src/axom/lumberjack/MPIUtility.hpp index fe7e7777bb..76a2667871 100644 --- a/src/axom/lumberjack/MPIUtility.hpp +++ b/src/axom/lumberjack/MPIUtility.hpp @@ -40,7 +40,7 @@ const char* mpiBlockingReceiveMessages(MPI_Comm comm); * MPI communication uses default LJ_Tag. ***************************************************************************** */ -const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag = 0); +const char* mpiNonBlockingReceiveMessages(MPI_Comm comm); /*! ***************************************************************************** @@ -58,8 +58,7 @@ const char* mpiNonBlockingReceiveMessages(MPI_Comm comm, int tag = 0); */ void mpiNonBlockingSendMessages(MPI_Comm comm, int destinationRank, - const char* packedMessagesToBeSent, - int tag = 0); + const char* packedMessagesToBeSent); } // end namespace lumberjack } // end namespace axom diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index 5fa68d3f2b..b310b58c16 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -13,10 +13,12 @@ ****************************************************************************** */ +#include #include #include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" #include "axom/lumberjack/MPIUtility.hpp" +#include namespace axom { @@ -24,27 +26,13 @@ namespace lumberjack { void NonCollectiveRootCommunicator::initialize(MPI_Comm comm, int ranksLimit) { - static int mpiTag = 32767; - m_mpiComm = comm; + MPI_Comm_dup(comm, &m_mpiComm); MPI_Comm_rank(m_mpiComm, &m_mpiCommRank); MPI_Comm_size(m_mpiComm, &m_mpiCommSize); m_ranksLimit = ranksLimit; - m_mpiTag = mpiTag; - /* - Each communicator that sends/receives messages non-collectively needs its - own mpiTag in order to not interfere with other communicators. - */ - if(mpiTag == std::numeric_limits::max()) - { - mpiTag = 32767; - } - else - { - ++mpiTag; - } } -void NonCollectiveRootCommunicator::finalize() { } +void NonCollectiveRootCommunicator::finalize() { MPI_Comm_free(&m_mpiComm); } int NonCollectiveRootCommunicator::rank() { return m_mpiCommRank; } @@ -67,7 +55,7 @@ void NonCollectiveRootCommunicator::push( bool receive_messages = true; while(receive_messages) { - currPackedMessages = mpiNonBlockingReceiveMessages(m_mpiComm, m_mpiTag); + currPackedMessages = mpiNonBlockingReceiveMessages(m_mpiComm); if(isPackedMessagesEmpty(currPackedMessages)) { @@ -92,7 +80,7 @@ void NonCollectiveRootCommunicator::push( { if(isPackedMessagesEmpty(packedMessagesToBeSent) == false) { - mpiNonBlockingSendMessages(m_mpiComm, 0, packedMessagesToBeSent, m_mpiTag); + mpiNonBlockingSendMessages(m_mpiComm, 0, packedMessagesToBeSent); } } } @@ -106,7 +94,5 @@ bool NonCollectiveRootCommunicator::isOutputNode() return false; } -int NonCollectiveRootCommunicator::mpiTag() const { return m_mpiTag; } - } // end namespace lumberjack } // end namespace axom diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp index 3a720ce534..e703595cfc 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.hpp @@ -122,19 +122,11 @@ class NonCollectiveRootCommunicator : public axom::lumberjack::Communicator */ bool isOutputNode(); - /*! - ***************************************************************************** - * \brief Returns the MPI tag used for this communicator. - ***************************************************************************** - */ - int mpiTag() const; - private: MPI_Comm m_mpiComm; int m_mpiCommRank; int m_mpiCommSize; int m_ranksLimit; - int m_mpiTag; }; } // end namespace lumberjack diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index 3cedd046c1..23b18c541e 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -4,6 +4,7 @@ // SPDX-License-Identifier: (BSD-3-Clause) #include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" +#include "axom/lumberjack/MPIUtility.hpp" #include "gtest/gtest.h" #include "mpi.h" @@ -87,12 +88,45 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) const int ranksLimit = 5; axom::lumberjack::NonCollectiveRootCommunicator c1; axom::lumberjack::NonCollectiveRootCommunicator c2; - + c1.initialize(MPI_COMM_WORLD, ranksLimit); c2.initialize(MPI_COMM_WORLD, ranksLimit); - // each communicator has unique MPI tag - EXPECT_NE(c1.mpiTag(), c2.mpiTag()); + std::vector receivedPackedMessages_c1; + std::vector receivedPackedMessages_c2; + const std::string c1_message = "c1"; + const std::string c2_message = "c2"; + + if (c1.rank() == 1 ) { + c1.push(c1_message.c_str(), receivedPackedMessages_c1); + } + + if (c2.rank() == 1 ) { + c2.push(c2_message.c_str(), receivedPackedMessages_c2); + } + + MPI_Barrier(MPI_COMM_WORLD); + + /* + Sending/receiving messages from c1 should not + interfere with sending/receiving from c2. + This tests that ordering of sends should not affect + the order of messages being received. + */ + + if (commSize > 1 && c1.rank() == 0) { + c1.push(nullptr, receivedPackedMessages_c1); + EXPECT_EQ(receivedPackedMessages_c1.size(), 1); + EXPECT_TRUE(!std::strcmp(receivedPackedMessages_c1[0], "c1")); + } + + MPI_Barrier(MPI_COMM_WORLD); + + if (commSize > 1 && c2.rank() == 0) { + c2.push(nullptr, receivedPackedMessages_c2); + EXPECT_EQ(receivedPackedMessages_c2.size(), 1); + EXPECT_TRUE(!std::strcmp(receivedPackedMessages_c2[0], "c2")); + } MPI_Barrier(MPI_COMM_WORLD); @@ -100,4 +134,19 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) c2.finalize(); MPI_Barrier(MPI_COMM_WORLD); + + // cleanup allocated memory from received messages + for(auto& rm : receivedPackedMessages_c1) + { + delete[] rm; + } + + for(auto& rm : receivedPackedMessages_c2) + { + delete[] rm; + } + + receivedPackedMessages_c1.clear(); + receivedPackedMessages_c2.clear(); + } From 3a3a52c2fc4432daacb8b4cfec6ff337b3d20423 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Tue, 28 Jan 2025 17:20:48 -0800 Subject: [PATCH 16/18] apply clang-format --- ...lumberjack_NonCollectiveRootCommunicator.hpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index 23b18c541e..8da83c704b 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -88,7 +88,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) const int ranksLimit = 5; axom::lumberjack::NonCollectiveRootCommunicator c1; axom::lumberjack::NonCollectiveRootCommunicator c2; - + c1.initialize(MPI_COMM_WORLD, ranksLimit); c2.initialize(MPI_COMM_WORLD, ranksLimit); @@ -97,11 +97,13 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) const std::string c1_message = "c1"; const std::string c2_message = "c2"; - if (c1.rank() == 1 ) { + if(c1.rank() == 1) + { c1.push(c1_message.c_str(), receivedPackedMessages_c1); } - if (c2.rank() == 1 ) { + if(c2.rank() == 1) + { c2.push(c2_message.c_str(), receivedPackedMessages_c2); } @@ -114,7 +116,8 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) the order of messages being received. */ - if (commSize > 1 && c1.rank() == 0) { + if(commSize > 1 && c1.rank() == 0) + { c1.push(nullptr, receivedPackedMessages_c1); EXPECT_EQ(receivedPackedMessages_c1.size(), 1); EXPECT_TRUE(!std::strcmp(receivedPackedMessages_c1[0], "c1")); @@ -122,7 +125,8 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) MPI_Barrier(MPI_COMM_WORLD); - if (commSize > 1 && c2.rank() == 0) { + if(commSize > 1 && c2.rank() == 0) + { c2.push(nullptr, receivedPackedMessages_c2); EXPECT_EQ(receivedPackedMessages_c2.size(), 1); EXPECT_TRUE(!std::strcmp(receivedPackedMessages_c2[0], "c2")); @@ -135,7 +139,7 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) MPI_Barrier(MPI_COMM_WORLD); - // cleanup allocated memory from received messages + // cleanup allocated memory from received messages for(auto& rm : receivedPackedMessages_c1) { delete[] rm; @@ -148,5 +152,4 @@ TEST(lumberjack_NonCollectiveRootCommunicator, multiple_communicators) receivedPackedMessages_c1.clear(); receivedPackedMessages_c2.clear(); - } From 5f416fdd506aca3fbe6c719126a225ae24fd6f0a Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Tue, 28 Jan 2025 17:29:54 -0800 Subject: [PATCH 17/18] removed unnecessary includes that my fancy IDE decided to add --- src/axom/lumberjack/MPIUtility.cpp | 1 - src/axom/lumberjack/NonCollectiveRootCommunicator.cpp | 2 -- .../tests/lumberjack_NonCollectiveRootCommunicator.hpp | 1 - 3 files changed, 4 deletions(-) diff --git a/src/axom/lumberjack/MPIUtility.cpp b/src/axom/lumberjack/MPIUtility.cpp index 420a5c42a2..dfd3ca6ef4 100644 --- a/src/axom/lumberjack/MPIUtility.cpp +++ b/src/axom/lumberjack/MPIUtility.cpp @@ -15,7 +15,6 @@ #include "axom/lumberjack/MPIUtility.hpp" #include -#include namespace axom { diff --git a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp index b310b58c16..de762d3042 100644 --- a/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp +++ b/src/axom/lumberjack/NonCollectiveRootCommunicator.cpp @@ -13,12 +13,10 @@ ****************************************************************************** */ -#include #include #include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" #include "axom/lumberjack/MPIUtility.hpp" -#include namespace axom { diff --git a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp index 8da83c704b..5fa0903ae0 100644 --- a/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp +++ b/src/axom/lumberjack/tests/lumberjack_NonCollectiveRootCommunicator.hpp @@ -4,7 +4,6 @@ // SPDX-License-Identifier: (BSD-3-Clause) #include "axom/lumberjack/NonCollectiveRootCommunicator.hpp" -#include "axom/lumberjack/MPIUtility.hpp" #include "gtest/gtest.h" #include "mpi.h" From ae837807dc37510c1f6b24c4f5c86425132e1f01 Mon Sep 17 00:00:00 2001 From: Guy Bergel Date: Wed, 29 Jan 2025 10:01:26 -0800 Subject: [PATCH 18/18] re-trigger tests