From feafdeccfcaf9eefa8642f57db5fc56a980f0a8a Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Tue, 28 Mar 2023 04:14:00 -0400 Subject: [PATCH 001/141] Add state to PCU mpi interface. This commit adds a MPI state that's separate from the global variables to the internal pcu_mpi interfaces. The user level PCU interface remains unchanged. However, this change lays the groundwork for PCU2 that will not make use of global state. This is important for some use cases like working around the spectrum-mpi comm_dup bug. The removal of global state also improves testability and modularity of this code. --- pcu/pcu.c | 75 ++++++++++---------- pcu/pcu_coll.c | 179 ++++++++++++++++++++++++------------------------ pcu/pcu_coll.h | 30 ++++---- pcu/pcu_io.c | 13 ++-- pcu/pcu_mpi.c | 53 +++++++------- pcu/pcu_mpi.h | 42 ++++++++---- pcu/pcu_msg.c | 39 +++++------ pcu/pcu_msg.h | 6 +- pcu/pcu_order.c | 8 +-- pcu/pcu_order.h | 2 +- pcu/pcu_pmpi.c | 81 +++++++++++----------- pcu/pcu_pmpi.h | 24 ++----- 12 files changed, 280 insertions(+), 272 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index b6c30f901..afc705504 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -33,7 +33,6 @@ #include #include "PCU.h" #include "pcu_msg.h" -#include "pcu_pmpi.h" #include "pcu_order.h" #include "noto_malloc.h" #include "reel.h" @@ -46,6 +45,7 @@ enum state { uninit, init }; static enum state global_state = uninit; static pcu_msg global_pmsg; +static pcu_mpi_t* global_mpi; static pcu_msg* get_msg() { @@ -61,8 +61,7 @@ int PCU_Comm_Init(void) { if (global_state != uninit) reel_fail("nested calls to Comm_Init"); - pcu_pmpi_init(MPI_COMM_WORLD); - pcu_set_mpi(&pcu_pmpi); + global_mpi = pcu_mpi_init(MPI_COMM_WORLD); pcu_make_msg(&global_pmsg); global_state = init; /* turn ordering on by default, call @@ -83,7 +82,7 @@ int PCU_Comm_Free(void) if (global_pmsg.order) pcu_order_free(global_pmsg.order); pcu_free_msg(&global_pmsg); - pcu_pmpi_finalize(); + pcu_mpi_finalize(&global_mpi); global_state = uninit; return PCU_SUCCESS; } @@ -101,7 +100,7 @@ int PCU_Comm_Self(void) { if (global_state == uninit) reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(); + return pcu_mpi_rank(global_mpi); } /** \brief Returns the number of threads in the program. @@ -112,7 +111,7 @@ int PCU_Comm_Peers(void) { if (global_state == uninit) reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(); + return pcu_mpi_size(global_mpi); } /** \brief Begins a PCU communication phase. @@ -125,7 +124,7 @@ void PCU_Comm_Begin(void) { if (global_state == uninit) reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(get_msg()); + pcu_msg_start(global_mpi, get_msg()); } /** \brief Packs data to be sent to \a to_rank. @@ -138,7 +137,7 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { if (global_state == uninit) reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) reel_fail("Invalid rank in Comm_Pack"); if ( size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); @@ -159,7 +158,7 @@ int PCU_Comm_Send(void) { if (global_state == uninit) reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(get_msg()); + pcu_msg_send(global_mpi, get_msg()); return PCU_SUCCESS; } @@ -180,8 +179,8 @@ bool PCU_Comm_Listen(void) reel_fail("Comm_Listen called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) - return pcu_order_receive(m->order, m); - return pcu_msg_receive(m); + return pcu_order_receive(global_mpi, m->order, m); + return pcu_msg_receive(global_mpi, m); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -250,7 +249,7 @@ void PCU_Barrier(void) { if (global_state == uninit) reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&(get_msg()->coll)); + pcu_barrier(global_mpi, &(get_msg()->coll)); } /** \brief Performs an Allreduce sum of double arrays. @@ -263,7 +262,7 @@ void PCU_Add_Doubles(double* p, size_t n) { if (global_state == uninit) reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); } double PCU_Add_Double(double x) @@ -280,7 +279,7 @@ void PCU_Min_Doubles(double* p, size_t n) { if (global_state == uninit) reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); } double PCU_Min_Double(double x) @@ -297,7 +296,7 @@ void PCU_Max_Doubles(double* p, size_t n) { if (global_state == uninit) reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); } double PCU_Max_Double(double x) @@ -314,7 +313,7 @@ void PCU_Add_Ints(int* p, size_t n) { if (global_state == uninit) reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); } int PCU_Add_Int(int x) @@ -331,7 +330,7 @@ void PCU_Add_Longs(long* p, size_t n) { if (global_state == uninit) reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); } long PCU_Add_Long(long x) @@ -348,7 +347,7 @@ void PCU_Add_SizeTs(size_t* p, size_t n) { if (global_state == uninit) reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); } size_t PCU_Add_SizeT(size_t x) @@ -364,7 +363,7 @@ size_t PCU_Add_SizeT(size_t x) void PCU_Min_SizeTs(size_t* p, size_t n) { if (global_state == uninit) reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); } size_t PCU_Min_SizeT(size_t x) { @@ -379,7 +378,7 @@ size_t PCU_Min_SizeT(size_t x) { void PCU_Max_SizeTs(size_t* p, size_t n) { if (global_state == uninit) reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); } size_t PCU_Max_SizeT(size_t x) { @@ -403,7 +402,7 @@ void PCU_Exscan_Ints(int* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(&(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -427,7 +426,7 @@ void PCU_Exscan_Longs(long* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(&(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -448,7 +447,7 @@ void PCU_Min_Ints(int* p, size_t n) { if (global_state == uninit) reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); } int PCU_Min_Int(int x) @@ -465,7 +464,7 @@ void PCU_Max_Ints(int* p, size_t n) { if (global_state == uninit) reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); } int PCU_Max_Int(int x) @@ -481,7 +480,7 @@ void PCU_Max_Longs(long* p, size_t n) { if (global_state == uninit) reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); } long PCU_Max_Long(long x) @@ -512,7 +511,7 @@ int PCU_Proc_Self(void) { if (global_state == uninit) reel_fail("Proc_Self called before Comm_Init"); - return pcu_pmpi_rank(); + return pcu_mpi_rank(global_mpi); } /** \brief Returns the number of processes. @@ -521,7 +520,7 @@ int PCU_Proc_Peers(void) { if (global_state == uninit) reel_fail("Proc_Peers called before Comm_Init"); - return pcu_pmpi_size(); + return pcu_mpi_size(global_mpi); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. @@ -530,7 +529,7 @@ int PCU_Comm_Rank(int* rank) { if (global_state == uninit) reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(); + *rank = pcu_mpi_rank(global_mpi); return PCU_SUCCESS; } @@ -539,7 +538,7 @@ int PCU_Comm_Size(int* size) { if (global_state == uninit) reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(); + *size = pcu_mpi_size(global_mpi); return PCU_SUCCESS; } @@ -556,7 +555,7 @@ int PCU_Comm_Start(PCU_Method method) (void)method; //warning silencer if (global_state == uninit) reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(get_msg()); + pcu_msg_start(global_mpi,get_msg()); return PCU_SUCCESS; } @@ -569,7 +568,7 @@ int PCU_Comm_Packed(int to_rank, size_t* size) { if (global_state == uninit) reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) reel_fail("Invalid rank in Comm_Packed"); *size = pcu_msg_packed(get_msg(),to_rank); return PCU_SUCCESS; @@ -588,7 +587,7 @@ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { if (global_state == uninit) reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) reel_fail("Invalid rank in Comm_Write"); pcu_msg* msg = get_msg(); PCU_MSG_PACK(msg,to_rank,size); @@ -741,13 +740,19 @@ void* PCU_Comm_Extract(size_t size) of this communicator, so you can safely get PCU to act on sub-groups of processes using this function. This call should be collective over all processes - in the previous communicator. + in the previous communicator. This is a very heavy weight function + and should be used sparingly. */ void PCU_Switch_Comm(MPI_Comm new_comm) { if (global_state == uninit) reel_fail("Switch_Comm called before Comm_Init"); - pcu_pmpi_switch(new_comm); + int result; + MPI_Comm_compare(new_comm, global_mpi->original_comm, &result); + if (result != MPI_IDENT) { + pcu_mpi_finalize(&global_mpi); + pcu_mpi_init(new_comm); + } } /** \brief Return the current MPI communicator @@ -759,7 +764,7 @@ MPI_Comm PCU_Get_Comm(void) { if (global_state == uninit) reel_fail("Get_Comm called before Comm_Init"); - return pcu_pmpi_comm(); + return global_mpi->original_comm; } /** \brief Return the time in seconds since some time in the past diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 7bdf8a221..39405aa6b 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_coll.h" -#include "pcu_pmpi.h" #include "reel.h" #include @@ -135,31 +134,31 @@ void pcu_add_longs(void* local, void* incoming, size_t size) /* initiates non-blocking calls for this communication step */ -static void begin_coll_step(pcu_coll* c) +static void begin_coll_step(pcu_mpi_t* mpi, pcu_coll* c) { - int action = c->pattern->action(c->bit); + int action = c->pattern->action(mpi, c->bit); if (action == pcu_coll_idle) return; - c->message.peer = c->pattern->peer(c->bit); + c->message.peer = c->pattern->peer(mpi, c->bit); if (action == pcu_coll_send) - pcu_mpi_send(&(c->message),pcu_coll_comm); + pcu_mpi_send(mpi, &(c->message),mpi->coll_comm); } /* tries to complete this communication step. Returns false if communication is not done, otherwise wraps up communication, merges if necessary, and returns true */ -static bool end_coll_step(pcu_coll* c) +static bool end_coll_step(pcu_mpi_t* mpi, pcu_coll* c) { - int action = c->pattern->action(c->bit); + int action = c->pattern->action(mpi, c->bit); if (action == pcu_coll_idle) return true; if (action == pcu_coll_send) - return pcu_mpi_done(&(c->message)); + return pcu_mpi_done(mpi, &(c->message)); pcu_message incoming; pcu_make_message(&incoming); - incoming.peer = c->pattern->peer(c->bit); - if ( ! pcu_mpi_receive(&incoming,pcu_coll_comm)) + incoming.peer = c->pattern->peer(mpi, c->bit); + if ( ! pcu_mpi_receive(mpi, &incoming,mpi->coll_comm)) return false; if (c->message.buffer.size != incoming.buffer.size) reel_fail("PCU unexpected incoming message.\n" @@ -169,7 +168,7 @@ static bool end_coll_step(pcu_coll* c) return true; } -void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m) +void pcu_make_coll(pcu_mpi_t* mpi, pcu_coll* c, pcu_pattern* p, pcu_merge* m) { c->pattern = p; c->merge = m; @@ -194,28 +193,28 @@ void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m) /* begins a non-blocking collective. The collective operation should be set with pcu_make_coll first. data[0..size] is the input/output local data */ -void pcu_begin_coll(pcu_coll* c, void* data, size_t size) +void pcu_begin_coll(pcu_mpi_t* mpi, pcu_coll* c, void* data, size_t size) { pcu_set_buffer(&(c->message.buffer),data,size); - c->bit = c->pattern->begin_bit(); - if (c->pattern->end_bit(c->bit)) + c->bit = c->pattern->begin_bit(mpi); + if (c->pattern->end_bit(mpi, c->bit)) return; - begin_coll_step(c); + begin_coll_step(mpi, c); } /* makes progress on a collective operation started by pcu_begin_coll. returns false if its done. */ -bool pcu_progress_coll(pcu_coll* c) +bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) { - if (c->pattern->end_bit(c->bit)) + if (c->pattern->end_bit(mpi, c->bit)) return false; - if (end_coll_step(c)) + if (end_coll_step(mpi, c)) { - c->bit = c->pattern->shift(c->bit); - if (c->pattern->end_bit(c->bit)) + c->bit = c->pattern->shift(mpi, c->bit); + if (c->pattern->end_bit(mpi, c->bit)) return false; - begin_coll_step(c); + begin_coll_step(mpi, c); } return true; } @@ -224,34 +223,34 @@ bool pcu_progress_coll(pcu_coll* c) then odd multiples of 2 into even ones, etc... until rank 0 has all inputs merged */ -static int reduce_begin_bit(void) +static int reduce_begin_bit(pcu_mpi_t*) { return 1; } -static bool reduce_end_bit(int bit) +static bool reduce_end_bit(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (rank==0) - return bit >= pcu_mpi_size(); + return bit >= pcu_mpi_size(mpi); return (bit>>1) & rank; } -static int reduce_peer(int bit) +static int reduce_peer(pcu_mpi_t* mpi, int bit) { - return pcu_mpi_rank() ^ bit; + return pcu_mpi_rank(mpi) ^ bit; } -static int reduce_action(int bit) +static int reduce_action(pcu_mpi_t* mpi, int bit) { - if (reduce_peer(bit) >= pcu_mpi_size()) + if (reduce_peer(mpi, bit) >= pcu_mpi_size(mpi)) return pcu_coll_idle; - if (bit & pcu_mpi_rank()) + if (bit & pcu_mpi_rank(mpi)) return pcu_coll_send; return pcu_coll_recv; } -static int reduce_shift(int bit) +static int reduce_shift(pcu_mpi_t*, int bit) { return bit << 1; } @@ -269,36 +268,36 @@ static pcu_pattern reduce = the pattern runs backwards and send/recv are flipped. */ -static int bcast_begin_bit(void) +static int bcast_begin_bit(pcu_mpi_t* mpi) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (rank == 0) - return 1 << ceil_log2(pcu_mpi_size()); + return 1 << ceil_log2(pcu_mpi_size(mpi)); int bit = 1; while ( ! (bit & rank)) bit <<= 1; return bit; } -static bool bcast_end_bit(int bit) +static bool bcast_end_bit(pcu_mpi_t *, int bit) { return bit == 0; } -static int bcast_peer(int bit) +static int bcast_peer(pcu_mpi_t * mpi, int bit) { - return pcu_mpi_rank() ^ bit; + return pcu_mpi_rank(mpi) ^ bit; } -static int bcast_action(int bit) +static int bcast_action(pcu_mpi_t* mpi, int bit) { - if (bcast_peer(bit) >= pcu_mpi_size()) + if (bcast_peer(mpi, bit) >= pcu_mpi_size(mpi)) return pcu_coll_idle; - if (bit & pcu_mpi_rank()) + if (bit & pcu_mpi_rank(mpi)) return pcu_coll_recv; return pcu_coll_send; } -static int bcast_shift(int bit) +static int bcast_shift(pcu_mpi_t *, int bit) { return bit >> 1; } @@ -319,14 +318,14 @@ static pcu_pattern bcast = "Parallel Prefix (Scan) Algorithms for MPI". */ -static int scan_up_begin_bit(void) +static int scan_up_begin_bit(pcu_mpi_t*) { return 1; } -static bool scan_up_end_bit(int bit) +static bool scan_up_end_bit(pcu_mpi_t* mpi, int bit) { - return bit == (1 << floor_log2(pcu_mpi_size())); + return bit == (1 << floor_log2(pcu_mpi_size(mpi))); } static bool scan_up_could_receive(int rank, int bit) @@ -345,34 +344,34 @@ static int scan_up_receiver_for(int rank, int bit) return rank + bit; } -static int scan_up_action(int bit) +static int scan_up_action(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if ((scan_up_could_receive(rank,bit))&& (0 <= scan_up_sender_for(rank,bit))) return pcu_coll_recv; int receiver = scan_up_receiver_for(rank,bit); - if ((receiver < pcu_mpi_size())&& + if ((receiver < pcu_mpi_size(mpi))&& (scan_up_could_receive(receiver,bit))) return pcu_coll_send; return pcu_coll_idle; } -static int scan_up_peer(int bit) +static int scan_up_peer(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); int sender = scan_up_sender_for(rank,bit); if ((scan_up_could_receive(rank,bit))&& (0 <= sender)) return sender; int receiver = scan_up_receiver_for(rank,bit); - if ((receiver < pcu_mpi_size())&& + if ((receiver < pcu_mpi_size(mpi))&& (scan_up_could_receive(receiver,bit))) return receiver; return -1; } -static int scan_up_shift(int bit) +static int scan_up_shift(pcu_mpi_t*, int bit) { return bit << 1; } @@ -386,12 +385,12 @@ static pcu_pattern scan_up = .shift = scan_up_shift, }; -static int scan_down_begin_bit(void) +static int scan_down_begin_bit(pcu_mpi_t* mpi) { - return 1 << floor_log2(pcu_mpi_size()); + return 1 << floor_log2(pcu_mpi_size(mpi)); } -static bool scan_down_end_bit(int bit) +static bool scan_down_end_bit(pcu_mpi_t*, int bit) { return bit == 1; } @@ -412,11 +411,11 @@ static int scan_down_sender_for(int rank, int bit) return rank - (bit >> 1); } -static int scan_down_action(int bit) +static int scan_down_action(pcu_mpi_t * mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if ((scan_down_could_send(rank,bit))&& - (scan_down_receiver_for(rank,bit) < pcu_mpi_size())) + (scan_down_receiver_for(rank,bit) < pcu_mpi_size(mpi))) return pcu_coll_send; int sender = scan_down_sender_for(rank,bit); if ((0 <= sender)&& @@ -425,13 +424,13 @@ static int scan_down_action(int bit) return pcu_coll_idle; } -static int scan_down_peer(int bit) +static int scan_down_peer(pcu_mpi_t * mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (scan_down_could_send(rank,bit)) { int receiver = scan_down_receiver_for(rank,bit); - if (receiver < pcu_mpi_size()) + if (receiver < pcu_mpi_size(mpi)) return receiver; } int sender = scan_down_sender_for(rank,bit); @@ -441,7 +440,7 @@ static int scan_down_peer(int bit) return -1; } -static int scan_down_shift(int bit) +static int scan_down_shift(pcu_mpi_t*, int bit) { return bit >> 1; } @@ -455,59 +454,59 @@ static pcu_pattern scan_down = .shift = scan_down_shift, }; -void pcu_reduce(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_reduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_make_coll(c,&reduce,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&reduce,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } -void pcu_bcast(pcu_coll* c, void* data, size_t size) +void pcu_bcast(pcu_mpi_t * mpi, pcu_coll* c, void* data, size_t size) { - pcu_make_coll(c,&bcast,pcu_merge_assign); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&bcast,pcu_merge_assign); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } -void pcu_allreduce(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_allreduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_reduce(c,m,data,size); - pcu_bcast(c,data,size); + pcu_reduce(mpi, c,m,data,size); + pcu_bcast(mpi, c,data,size); } -void pcu_scan(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_scan(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_make_coll(c,&scan_up,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); - pcu_make_coll(c,&scan_down,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&scan_up,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); + pcu_make_coll(mpi, c,&scan_down,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } /* a barrier is just an allreduce of nothing in particular */ -void pcu_begin_barrier(pcu_coll* c) +void pcu_begin_barrier(pcu_mpi_t* mpi, pcu_coll* c) { - pcu_make_coll(c,&reduce,pcu_merge_assign); - pcu_begin_coll(c,NULL,0); + pcu_make_coll(mpi, c,&reduce,pcu_merge_assign); + pcu_begin_coll(mpi, c,NULL,0); } -bool pcu_barrier_done(pcu_coll* c) +bool pcu_barrier_done(pcu_mpi_t* mpi, pcu_coll* c) { if (c->pattern == &reduce) - if ( ! pcu_progress_coll(c)) + if ( ! pcu_progress_coll(mpi, c)) { - pcu_make_coll(c,&bcast,pcu_merge_assign); - pcu_begin_coll(c,c->message.buffer.start,c->message.buffer.size); + pcu_make_coll(mpi, c,&bcast,pcu_merge_assign); + pcu_begin_coll(mpi, c,c->message.buffer.start,c->message.buffer.size); } if (c->pattern == &bcast) - if ( ! pcu_progress_coll(c)) + if ( ! pcu_progress_coll(mpi,c)) return true; return false; } -void pcu_barrier(pcu_coll* c) +void pcu_barrier(pcu_mpi_t* mpi, pcu_coll* c) { - pcu_begin_barrier(c); - while( ! pcu_barrier_done(c)); + pcu_begin_barrier(mpi, c); + while( ! pcu_barrier_done(mpi, c)); } diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index 11f204ee8..9c89d64f9 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -57,11 +57,11 @@ enum */ typedef struct { - int (*begin_bit)(void); //initialize state bit - bool (*end_bit)(int bit); //return true if bit is one past the last - int (*action)(int bit); //return action enum for this step - int (*peer)(int bit); //return the peer to communicate with - int (*shift)(int bit); //shift the bit up or down + int (*begin_bit)(pcu_mpi_t*); //initialize state bit + bool (*end_bit)(pcu_mpi_t*, int bit); //return true if bit is one past the last + int (*action)(pcu_mpi_t*, int bit); //return action enum for this step + int (*peer)(pcu_mpi_t*, int bit); //return the peer to communicate with + int (*shift)(pcu_mpi_t*, int bit); //shift the bit up or down } pcu_pattern; /* The pcu_coll object stores the state of a non-blocking @@ -74,18 +74,18 @@ typedef struct int bit; //pattern's state bit } pcu_coll; -void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m); -void pcu_begin_coll(pcu_coll* c, void* data, size_t size); +void pcu_make_coll(pcu_mpi_t *, pcu_coll* c, pcu_pattern* p, pcu_merge* m); +void pcu_begin_coll(pcu_mpi_t *, pcu_coll* c, void* data, size_t size); //returns false when done -bool pcu_progress_coll(pcu_coll* c); +bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c); -void pcu_reduce(pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_bcast(pcu_coll* c, void* data, size_t size); -void pcu_allreduce(pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_scan(pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_reduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_bcast(pcu_mpi_t*, pcu_coll* c, void* data, size_t size); +void pcu_allreduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_scan(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_begin_barrier(pcu_coll* c); -bool pcu_barrier_done(pcu_coll* c); -void pcu_barrier(pcu_coll* c); +void pcu_begin_barrier(pcu_mpi_t*,pcu_coll* c); +bool pcu_barrier_done(pcu_mpi_t*, pcu_coll* c); +void pcu_barrier(pcu_mpi_t*, pcu_coll* c); #endif //PCU_COLL_H diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index a706f6155..a89f366bd 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,16 +8,16 @@ *******************************************************************************/ #include "pcu_io.h" +#include "PCU.h" #include "noto_malloc.h" +#include "pcu_buffer.h" +#include "pcu_util.h" #include "reel.h" -#include "pcu_mpi.h" -#include "PCU.h" +#include #include -#include #include -#include "pcu_util.h" +#include #include -#include #ifdef PCU_BZIP #include @@ -361,7 +361,8 @@ FILE* pcu_open_parallel(const char* prefix, const char* ext) static const size_t max_rank_chars = 10; size_t path_size = strlen(prefix) + max_rank_chars + strlen(ext) + 1; char* path = noto_malloc(path_size); - int rank = pcu_mpi_rank(); + int rank; + PCU_Comm_Rank(&rank); snprintf(path,path_size,"%s%d.%s",prefix,rank,ext); FILE* file = fopen(path, "w"); noto_free(path); diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index 75009d277..c5dea8baa 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -9,8 +9,7 @@ *******************************************************************************/ #include "pcu_mpi.h" #include "pcu_util.h" - -static pcu_mpi* global_mpi; +#include "pcu_pmpi.h" void pcu_make_message(pcu_message* m) { @@ -22,47 +21,49 @@ void pcu_free_message(pcu_message* m) pcu_free_buffer(&(m->buffer)); } -void pcu_set_mpi(pcu_mpi* m) +int pcu_mpi_size(pcu_mpi_t* self) { - global_mpi = m; + return pcu_pmpi_size(self); } -pcu_mpi* pcu_get_mpi(void) +int pcu_mpi_rank(pcu_mpi_t* self) { - return global_mpi; + return pcu_pmpi_rank(self); } -int pcu_mpi_size(void) -{ - return global_mpi->size(); -} - -int pcu_mpi_rank(void) -{ - return global_mpi->rank(); -} - -static void check_rank(int rank) +static void check_rank(pcu_mpi_t* self, int rank) { (void)rank; PCU_ALWAYS_ASSERT(0 <= rank); - PCU_ALWAYS_ASSERT(rank < pcu_mpi_size()); + PCU_ALWAYS_ASSERT(rank < pcu_mpi_size(self)); } -void pcu_mpi_send(pcu_message* m, MPI_Comm comm) +void pcu_mpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - check_rank(m->peer); - global_mpi->send(m,comm); + check_rank(self, m->peer); + // verify that the communicator is one of the user or collective communicators + //int user_result, coll_result; + //MPI_Comm_compare(comm, self->user_comm, &user_result); + //MPI_Comm_compare(comm, self->coll_comm, &coll_result); + //PCU_ALWAYS_ASSERT(user_result == MPI_IDENT || coll_result == MPI_IDENT); + PCU_ALWAYS_ASSERT(comm == self->user_comm || comm == self->coll_comm); + pcu_pmpi_send(self, m, comm); } -bool pcu_mpi_done(pcu_message* m) +bool pcu_mpi_done(pcu_mpi_t* self, pcu_message* m) { - return global_mpi->done(m); + return pcu_pmpi_done(self, m); } -bool pcu_mpi_receive(pcu_message* m, MPI_Comm comm) +bool pcu_mpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { if (m->peer != MPI_ANY_SOURCE) - check_rank(m->peer); - return global_mpi->receive(m,comm); + check_rank(self, m->peer); + return pcu_pmpi_receive(self, m, comm); +} +pcu_mpi_t* pcu_mpi_init(MPI_Comm comm) { + return pcu_pmpi_init(comm); +} +void pcu_mpi_finalize(pcu_mpi_t** mpi) { + pcu_pmpi_finalize(mpi); } diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 052920d04..3c2dcc1f9 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -23,21 +23,35 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); +struct pcu_mpi_t_t; typedef struct { - int (*size)(void); - int (*rank)(void); - void (*send)(pcu_message* m, MPI_Comm comm); - bool (*done)(pcu_message* m); - bool (*receive)(pcu_message* m, MPI_Comm comm); -} pcu_mpi; - -void pcu_set_mpi(pcu_mpi* m); -pcu_mpi* pcu_get_mpi(void); -int pcu_mpi_size(void); -int pcu_mpi_rank(void); -void pcu_mpi_send(pcu_message* m, MPI_Comm comm); -bool pcu_mpi_done(pcu_message* m); -bool pcu_mpi_receive(pcu_message* m, MPI_Comm comm); + int (*size)(struct pcu_mpi_t_t*); + int (*rank)(struct pcu_mpi_t_t*); + void (*send)(struct pcu_mpi_t_t *, pcu_message* m, MPI_Comm comm); + bool (*done)(struct pcu_mpi_t_t*, pcu_message* m); + bool (*receive)(struct pcu_mpi_t_t*, pcu_message* m, MPI_Comm comm); + void (*construct)(struct pcu_mpi_t_t*, MPI_Comm comm); + void (*destruct)(struct pcu_mpi_t_t*); +} pcu_mpi_vtable; + +struct pcu_mpi_t_t +{ + pcu_mpi_vtable const* vtable; + MPI_Comm original_comm; + MPI_Comm user_comm; + MPI_Comm coll_comm; + int rank; + int size; +}; +typedef struct pcu_mpi_t_t pcu_mpi_t; + +int pcu_mpi_size(pcu_mpi_t*); +int pcu_mpi_rank(pcu_mpi_t*); +void pcu_mpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_mpi_done(pcu_mpi_t*, pcu_message* m); +bool pcu_mpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +pcu_mpi_t* pcu_mpi_init(MPI_Comm comm); +void pcu_mpi_finalize(pcu_mpi_t**); #endif diff --git a/pcu/pcu_msg.c b/pcu/pcu_msg.c index 2b4705890..538310c30 100644 --- a/pcu/pcu_msg.c +++ b/pcu/pcu_msg.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_msg.h" -#include "pcu_pmpi.h" #include "noto_malloc.h" #include "reel.h" #include @@ -87,14 +86,14 @@ static void free_peers(pcu_aa_tree* t) pcu_make_aa(t); } -void pcu_msg_start(pcu_msg* m) +void pcu_msg_start(pcu_mpi_t* mpi, pcu_msg* m) { if (m->state != idle_state) reel_fail("PCU_Comm_Begin called at the wrong time"); /* this barrier ensures no one starts a new superstep while others are receiving in the past superstep. It is the only blocking call in the pcu_msg system. */ - pcu_barrier(&(m->coll)); + pcu_barrier(mpi, &(m->coll)); m->state = pack_state; } @@ -143,49 +142,49 @@ size_t pcu_msg_packed(pcu_msg* m, int id) return peer->message.buffer.size; } -static void send_peers(pcu_aa_tree t) +static void send_peers(pcu_mpi_t* mpi, pcu_aa_tree t) { if (pcu_aa_empty(t)) return; pcu_msg_peer* peer; peer = (pcu_msg_peer*)t; - pcu_mpi_send(&(peer->message),pcu_user_comm); - send_peers(t->left); - send_peers(t->right); + pcu_mpi_send(mpi, &(peer->message),mpi->user_comm); + send_peers(mpi, t->left); + send_peers(mpi, t->right); } -void pcu_msg_send(pcu_msg* m) +void pcu_msg_send(pcu_mpi_t* mpi, pcu_msg* m) { if (m->state != pack_state) reel_fail("PCU_Comm_Send called at the wrong time"); - send_peers(m->peers); + send_peers(mpi, m->peers); m->state = send_recv_state; } -static bool done_sending_peers(pcu_aa_tree t) +static bool done_sending_peers(pcu_mpi_t* mpi, pcu_aa_tree t) { if (pcu_aa_empty(t)) return true; pcu_msg_peer* peer; peer = (pcu_msg_peer*)t; - return pcu_mpi_done(&(peer->message)) - && done_sending_peers(t->left) - && done_sending_peers(t->right); + return pcu_mpi_done(mpi, &(peer->message)) + && done_sending_peers(mpi, t->left) + && done_sending_peers(mpi, t->right); } -static bool receive_global(pcu_msg* m) +static bool receive_global(pcu_mpi_t* mpi, pcu_msg* m) { m->received.peer = MPI_ANY_SOURCE; - while ( ! pcu_mpi_receive(&(m->received),pcu_user_comm)) + while ( ! pcu_mpi_receive(mpi, &(m->received),mpi->user_comm)) { if (m->state == send_recv_state) - if (done_sending_peers(m->peers)) + if (done_sending_peers(mpi, m->peers)) { - pcu_begin_barrier(&(m->coll)); + pcu_begin_barrier(mpi, &(m->coll)); m->state = recv_state; } if (m->state == recv_state) - if (pcu_barrier_done(&(m->coll))) + if (pcu_barrier_done(mpi, &(m->coll))) return false; } return true; @@ -197,14 +196,14 @@ static void free_comm(pcu_msg* m) pcu_free_message(&(m->received)); } -bool pcu_msg_receive(pcu_msg* m) +bool pcu_msg_receive(pcu_mpi_t* mpi, pcu_msg* m) { if ((m->state != send_recv_state)&& (m->state != recv_state)) reel_fail("PCU_Comm_Receive called at the wrong time"); if ( ! pcu_msg_unpacked(m)) reel_fail("PCU_Comm_Receive called before previous message unpacked"); - if (receive_global(m)) + if (receive_global(mpi, m)) { pcu_begin_buffer(&(m->received.buffer)); return true; diff --git a/pcu/pcu_msg.h b/pcu/pcu_msg.h index 48713e80a..0289d0d16 100644 --- a/pcu/pcu_msg.h +++ b/pcu/pcu_msg.h @@ -45,13 +45,13 @@ struct pcu_msg_struct typedef struct pcu_msg_struct pcu_msg; void pcu_make_msg(pcu_msg* m); -void pcu_msg_start(pcu_msg* b); +void pcu_msg_start(pcu_mpi_t*, pcu_msg* b); void* pcu_msg_pack(pcu_msg* m, int id, size_t size); #define PCU_MSG_PACK(m,id,o) \ memcpy(pcu_msg_pack(m,id,sizeof(o)),&(o),sizeof(o)) size_t pcu_msg_packed(pcu_msg* m, int id); -void pcu_msg_send(pcu_msg* m); -bool pcu_msg_receive(pcu_msg* m); +void pcu_msg_send(pcu_mpi_t *mpi, pcu_msg* m); +bool pcu_msg_receive(pcu_mpi_t* mpi, pcu_msg* m); void* pcu_msg_unpack(pcu_msg* m, size_t size); #define PCU_MSG_UNPACK(m,o) \ memcpy(&(o),pcu_msg_unpack(m,sizeof(o)),sizeof(o)) diff --git a/pcu/pcu_order.c b/pcu/pcu_order.c index 1099e2d80..d0989a056 100644 --- a/pcu/pcu_order.c +++ b/pcu/pcu_order.c @@ -102,10 +102,10 @@ static void fill(pcu_order o, pcu_aa_tree t) fill(o, t->right); } -static void prepare(pcu_order o, pcu_msg* t) +static void prepare(pcu_mpi_t* mpi, pcu_order o, pcu_msg* t) { struct message* m; - while (pcu_msg_receive(t)) { + while (pcu_msg_receive(mpi, t)) { m = take_message(t); pcu_aa_insert(&m->node, &o->tree, message_less); } @@ -117,10 +117,10 @@ static void prepare(pcu_order o, pcu_msg* t) o->ready = true; } -bool pcu_order_receive(pcu_order o, pcu_msg* m) +bool pcu_order_receive(pcu_mpi_t* mpi, pcu_order o, pcu_msg* m) { if (!o->ready) - prepare(o, m); + prepare(mpi, o, m); o->at++; if (o->at == o->count) { dtor_order(o); diff --git a/pcu/pcu_order.h b/pcu/pcu_order.h index b762a1752..26e144885 100644 --- a/pcu/pcu_order.h +++ b/pcu/pcu_order.h @@ -17,7 +17,7 @@ typedef struct pcu_order_struct* pcu_order; pcu_order pcu_order_new(void); void pcu_order_free(pcu_order o); -bool pcu_order_receive(pcu_order o, pcu_msg* m); +bool pcu_order_receive(pcu_mpi_t*, pcu_order o, pcu_msg* m); void* pcu_order_unpack(pcu_order o, size_t size); bool pcu_order_unpacked(pcu_order o); int pcu_order_received_from(pcu_order o); diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 814828a7e..44b53c762 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -13,51 +13,62 @@ #include #include -static int global_size; -static int global_rank; +void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); +bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); +void pcu_pmpi_destruct(pcu_mpi_t* self); +void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm); -MPI_Comm original_comm; -MPI_Comm pcu_user_comm; -MPI_Comm pcu_coll_comm; - -pcu_mpi pcu_pmpi = +static pcu_mpi_vtable pcu_pmpi_vtable = { .size = pcu_pmpi_size, .rank = pcu_pmpi_rank, .send = pcu_pmpi_send, .done = pcu_pmpi_done, - .receive = pcu_pmpi_receive }; + .receive = pcu_pmpi_receive, + .construct = pcu_pmpi_construct, + .destruct = pcu_pmpi_destruct }; -void pcu_pmpi_init(MPI_Comm comm) +pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) { - original_comm = comm; - MPI_Comm_dup(comm,&pcu_user_comm); - MPI_Comm_dup(comm,&pcu_coll_comm); - MPI_Comm_size(comm,&global_size); - MPI_Comm_rank(comm,&global_rank); + pcu_mpi_t* m = (pcu_mpi_t*)malloc(sizeof(pcu_mpi_t)); + pcu_pmpi_construct(m,comm); + return m; } -void pcu_pmpi_finalize(void) +void pcu_pmpi_finalize(pcu_mpi_t** m) { - MPI_Comm_free(&pcu_user_comm); - MPI_Comm_free(&pcu_coll_comm); + (*m)->vtable->destruct(*m); + free(*m); + *m = NULL; +} +void pcu_pmpi_destruct(pcu_mpi_t* self) { + MPI_Comm_free(&(self->user_comm)); + MPI_Comm_free(&(self->coll_comm)); +} +void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { + self->original_comm = comm; + MPI_Comm_dup(comm,&(self->user_comm)); + MPI_Comm_dup(comm,&(self->coll_comm)); + MPI_Comm_size(comm,&(self->size)); + MPI_Comm_rank(comm,&(self->rank)); + self->vtable = &pcu_pmpi_vtable; } -int pcu_pmpi_size(void) +int pcu_pmpi_size(pcu_mpi_t* self) { - return global_size; + return self->size; } -int pcu_pmpi_rank(void) +int pcu_pmpi_rank(pcu_mpi_t* self) { - return global_rank; + return self->rank; } -void pcu_pmpi_send(pcu_message* m, MPI_Comm comm) +void pcu_pmpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - pcu_pmpi_send2(m,0,comm); + pcu_pmpi_send2(self, m,0,comm); } -void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm) +void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { if( m->buffer.size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR PCU message size exceeds INT_MAX... exiting\n"); @@ -73,19 +84,19 @@ void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm) &(m->request)); } -bool pcu_pmpi_done(pcu_message* m) +bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) { int flag; MPI_Test(&(m->request),&flag,MPI_STATUS_IGNORE); return flag; } -bool pcu_pmpi_receive(pcu_message* m, MPI_Comm comm) +bool pcu_pmpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - return pcu_pmpi_receive2(m,0,comm); + return pcu_pmpi_receive2(self, m,0,comm); } -bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm) +bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { MPI_Status status; int flag; @@ -105,16 +116,4 @@ bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm) comm, MPI_STATUS_IGNORE); return true; -} - -void pcu_pmpi_switch(MPI_Comm new_comm) -{ - pcu_pmpi_finalize(); - pcu_pmpi_init(new_comm); -} - -MPI_Comm pcu_pmpi_comm(void) -{ - return original_comm; -} - +} \ No newline at end of file diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index c6707d23e..cdfbbf699 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -14,22 +14,12 @@ #include -void pcu_pmpi_init(MPI_Comm comm); -void pcu_pmpi_finalize(void); -int pcu_pmpi_size(void); -int pcu_pmpi_rank(void); -void pcu_pmpi_send(pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(pcu_message* m, MPI_Comm comm); -void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_done(pcu_message* m); - -void pcu_pmpi_switch(MPI_Comm new_comm); -MPI_Comm pcu_pmpi_comm(void); - -extern pcu_mpi pcu_pmpi; - -extern MPI_Comm pcu_user_comm; -extern MPI_Comm pcu_coll_comm; +pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm); +void pcu_pmpi_finalize(pcu_mpi_t** m); +int pcu_pmpi_size(pcu_mpi_t* self); +int pcu_pmpi_rank(pcu_mpi_t* self); +void pcu_pmpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_done(pcu_mpi_t*, pcu_message* m); #endif From 7c390ee91186c13bb014a602db50d8ab43e40c11 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Tue, 28 Mar 2023 16:01:41 -0400 Subject: [PATCH 002/141] Compile on gcc+fix bug --- pcu/pcu.c | 2 +- pcu/pcu_coll.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index afc705504..89fad1858 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -751,7 +751,7 @@ void PCU_Switch_Comm(MPI_Comm new_comm) MPI_Comm_compare(new_comm, global_mpi->original_comm, &result); if (result != MPI_IDENT) { pcu_mpi_finalize(&global_mpi); - pcu_mpi_init(new_comm); + global_mpi = pcu_mpi_init(new_comm); } } diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 39405aa6b..225113b13 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -223,7 +223,7 @@ bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) then odd multiples of 2 into even ones, etc... until rank 0 has all inputs merged */ -static int reduce_begin_bit(pcu_mpi_t*) +static int reduce_begin_bit(pcu_mpi_t* mpi) { return 1; } @@ -250,7 +250,7 @@ static int reduce_action(pcu_mpi_t* mpi, int bit) return pcu_coll_recv; } -static int reduce_shift(pcu_mpi_t*, int bit) +static int reduce_shift(pcu_mpi_t* mpi, int bit) { return bit << 1; } @@ -278,7 +278,7 @@ static int bcast_begin_bit(pcu_mpi_t* mpi) return bit; } -static bool bcast_end_bit(pcu_mpi_t *, int bit) +static bool bcast_end_bit(pcu_mpi_t * mpi, int bit) { return bit == 0; } @@ -297,7 +297,7 @@ static int bcast_action(pcu_mpi_t* mpi, int bit) return pcu_coll_send; } -static int bcast_shift(pcu_mpi_t *, int bit) +static int bcast_shift(pcu_mpi_t * mpi, int bit) { return bit >> 1; } @@ -318,7 +318,7 @@ static pcu_pattern bcast = "Parallel Prefix (Scan) Algorithms for MPI". */ -static int scan_up_begin_bit(pcu_mpi_t*) +static int scan_up_begin_bit(pcu_mpi_t* mpi) { return 1; } @@ -371,7 +371,7 @@ static int scan_up_peer(pcu_mpi_t* mpi, int bit) return -1; } -static int scan_up_shift(pcu_mpi_t*, int bit) +static int scan_up_shift(pcu_mpi_t* mpi, int bit) { return bit << 1; } @@ -390,7 +390,7 @@ static int scan_down_begin_bit(pcu_mpi_t* mpi) return 1 << floor_log2(pcu_mpi_size(mpi)); } -static bool scan_down_end_bit(pcu_mpi_t*, int bit) +static bool scan_down_end_bit(pcu_mpi_t* mpi, int bit) { return bit == 1; } @@ -440,7 +440,7 @@ static int scan_down_peer(pcu_mpi_t * mpi, int bit) return -1; } -static int scan_down_shift(pcu_mpi_t*, int bit) +static int scan_down_shift(pcu_mpi_t* mpi, int bit) { return bit >> 1; } From a4bbbfbdd208ef318a862837c49736c09090340a Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 02:05:18 -0400 Subject: [PATCH 003/141] remove virtual function table for pcu_mpi Throughout the code base it was assumed that the mpi functions were implemented through pmpi and so no other implementation really could have been implemented. This change just removes the complexity from having the vtable. --- pcu/pcu_mpi.h | 16 +--------------- pcu/pcu_pmpi.c | 13 ++----------- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 3c2dcc1f9..1bed6d6c7 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -23,28 +23,14 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); -struct pcu_mpi_t_t; typedef struct { - int (*size)(struct pcu_mpi_t_t*); - int (*rank)(struct pcu_mpi_t_t*); - void (*send)(struct pcu_mpi_t_t *, pcu_message* m, MPI_Comm comm); - bool (*done)(struct pcu_mpi_t_t*, pcu_message* m); - bool (*receive)(struct pcu_mpi_t_t*, pcu_message* m, MPI_Comm comm); - void (*construct)(struct pcu_mpi_t_t*, MPI_Comm comm); - void (*destruct)(struct pcu_mpi_t_t*); -} pcu_mpi_vtable; - -struct pcu_mpi_t_t -{ - pcu_mpi_vtable const* vtable; MPI_Comm original_comm; MPI_Comm user_comm; MPI_Comm coll_comm; int rank; int size; -}; -typedef struct pcu_mpi_t_t pcu_mpi_t; +} pcu_mpi_t; int pcu_mpi_size(pcu_mpi_t*); int pcu_mpi_rank(pcu_mpi_t*); diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 44b53c762..e8f546d70 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -18,14 +18,6 @@ bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); void pcu_pmpi_destruct(pcu_mpi_t* self); void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm); -static pcu_mpi_vtable pcu_pmpi_vtable = -{ .size = pcu_pmpi_size, - .rank = pcu_pmpi_rank, - .send = pcu_pmpi_send, - .done = pcu_pmpi_done, - .receive = pcu_pmpi_receive, - .construct = pcu_pmpi_construct, - .destruct = pcu_pmpi_destruct }; pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) { @@ -36,7 +28,7 @@ pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) void pcu_pmpi_finalize(pcu_mpi_t** m) { - (*m)->vtable->destruct(*m); + pcu_pmpi_destruct(*m); free(*m); *m = NULL; } @@ -50,7 +42,6 @@ void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { MPI_Comm_dup(comm,&(self->coll_comm)); MPI_Comm_size(comm,&(self->size)); MPI_Comm_rank(comm,&(self->rank)); - self->vtable = &pcu_pmpi_vtable; } int pcu_pmpi_size(pcu_mpi_t* self) @@ -116,4 +107,4 @@ bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) comm, MPI_STATUS_IGNORE); return true; -} \ No newline at end of file +} From baf937028c73560222e1d46fe1726cfaf6d7b7da Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 02:52:17 -0400 Subject: [PATCH 004/141] Put PCU state data into a structure --- pcu/pcu.c | 178 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 91 insertions(+), 87 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index 89fad1858..8dd48f0a2 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -42,14 +42,18 @@ #include /*INT_MAX*/ #include /*abort*/ -enum state { uninit, init }; -static enum state global_state = uninit; -static pcu_msg global_pmsg; -static pcu_mpi_t* global_mpi; +typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; +typedef struct { + pcu_state_enum state; + pcu_msg pmsg; + pcu_mpi_t* mpi; +} pcu_t; + +static pcu_t global_pcu = {.state=pcu_state_uninit}; static pcu_msg* get_msg() { - return &global_pmsg; + return &(global_pcu.pmsg); } /** \brief Initializes the PCU library. @@ -59,11 +63,11 @@ static pcu_msg* get_msg() */ int PCU_Comm_Init(void) { - if (global_state != uninit) + if (global_pcu.state != pcu_state_uninit) reel_fail("nested calls to Comm_Init"); - global_mpi = pcu_mpi_init(MPI_COMM_WORLD); - pcu_make_msg(&global_pmsg); - global_state = init; + global_pcu.mpi = pcu_mpi_init(MPI_COMM_WORLD); + pcu_make_msg(&(global_pcu.pmsg)); + global_pcu.state = pcu_state_init; /* turn ordering on by default, call PCU_Comm_Order(false) after PCU_Comm_Init to disable this */ @@ -77,13 +81,13 @@ int PCU_Comm_Init(void) */ int PCU_Comm_Free(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Free called before Comm_Init"); - if (global_pmsg.order) - pcu_order_free(global_pmsg.order); - pcu_free_msg(&global_pmsg); - pcu_mpi_finalize(&global_mpi); - global_state = uninit; + if (global_pcu.pmsg.order) + pcu_order_free(global_pcu.pmsg.order); + pcu_free_msg(&global_pcu.pmsg); + pcu_mpi_finalize(&global_pcu.mpi); + global_pcu.state = pcu_state_uninit; return PCU_SUCCESS; } @@ -98,9 +102,9 @@ int PCU_Comm_Free(void) */ int PCU_Comm_Self(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(global_mpi); + return pcu_mpi_rank(global_pcu.mpi); } /** \brief Returns the number of threads in the program. @@ -109,9 +113,9 @@ int PCU_Comm_Self(void) */ int PCU_Comm_Peers(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(global_mpi); + return pcu_mpi_size(global_pcu.mpi); } /** \brief Begins a PCU communication phase. @@ -122,9 +126,9 @@ int PCU_Comm_Peers(void) */ void PCU_Comm_Begin(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(global_mpi, get_msg()); + pcu_msg_start(global_pcu.mpi, get_msg()); } /** \brief Packs data to be sent to \a to_rank. @@ -135,9 +139,9 @@ void PCU_Comm_Begin(void) */ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) reel_fail("Invalid rank in Comm_Pack"); if ( size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); @@ -156,9 +160,9 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) */ int PCU_Comm_Send(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(global_mpi, get_msg()); + pcu_msg_send(global_pcu.mpi, get_msg()); return PCU_SUCCESS; } @@ -175,12 +179,12 @@ int PCU_Comm_Send(void) */ bool PCU_Comm_Listen(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Listen called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) - return pcu_order_receive(global_mpi, m->order, m); - return pcu_msg_receive(global_mpi, m); + return pcu_order_receive(global_pcu.mpi, m->order, m); + return pcu_msg_receive(global_pcu.mpi, m); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -188,7 +192,7 @@ bool PCU_Comm_Listen(void) */ int PCU_Comm_Sender(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Sender called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -201,7 +205,7 @@ int PCU_Comm_Sender(void) */ bool PCU_Comm_Unpacked(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Unpacked called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -221,7 +225,7 @@ bool PCU_Comm_Unpacked(void) */ int PCU_Comm_Unpack(void* data, size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Unpack called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -233,7 +237,7 @@ int PCU_Comm_Unpack(void* data, size_t size) void PCU_Comm_Order(bool on) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Order called before Comm_Init"); pcu_msg* m = get_msg(); if (on && (!m->order)) @@ -247,9 +251,9 @@ void PCU_Comm_Order(bool on) /** \brief Blocking barrier over all threads. */ void PCU_Barrier(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Barrier called before Comm_Init"); - pcu_barrier(global_mpi, &(get_msg()->coll)); + pcu_barrier(global_pcu.mpi, &(get_msg()->coll)); } /** \brief Performs an Allreduce sum of double arrays. @@ -260,9 +264,9 @@ void PCU_Barrier(void) */ void PCU_Add_Doubles(double* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); } double PCU_Add_Double(double x) @@ -277,9 +281,9 @@ double PCU_Add_Double(double x) */ void PCU_Min_Doubles(double* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); } double PCU_Min_Double(double x) @@ -294,9 +298,9 @@ double PCU_Min_Double(double x) */ void PCU_Max_Doubles(double* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); } double PCU_Max_Double(double x) @@ -311,9 +315,9 @@ double PCU_Max_Double(double x) */ void PCU_Add_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); } int PCU_Add_Int(int x) @@ -328,9 +332,9 @@ int PCU_Add_Int(int x) */ void PCU_Add_Longs(long* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); } long PCU_Add_Long(long x) @@ -345,9 +349,9 @@ long PCU_Add_Long(long x) */ void PCU_Add_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); } size_t PCU_Add_SizeT(size_t x) @@ -361,9 +365,9 @@ size_t PCU_Add_SizeT(size_t x) /** \brief Performs an Allreduce minimum of size_t unsigned integers */ void PCU_Min_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); } size_t PCU_Min_SizeT(size_t x) { @@ -376,9 +380,9 @@ size_t PCU_Min_SizeT(size_t x) { /** \brief Performs an Allreduce maximum of size_t unsigned integers */ void PCU_Max_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); } size_t PCU_Max_SizeT(size_t x) { @@ -396,13 +400,13 @@ size_t PCU_Max_SizeT(size_t x) { */ void PCU_Exscan_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Exscan_Ints called before Comm_Init"); int* originals; NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -420,13 +424,13 @@ int PCU_Exscan_Int(int x) /** \brief See PCU_Exscan_Ints */ void PCU_Exscan_Longs(long* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Exscan_Longs called before Comm_Init"); long* originals; NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -445,9 +449,9 @@ long PCU_Exscan_Long(long x) */ void PCU_Min_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); } int PCU_Min_Int(int x) @@ -462,9 +466,9 @@ int PCU_Min_Int(int x) */ void PCU_Max_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); } int PCU_Max_Int(int x) @@ -478,9 +482,9 @@ int PCU_Max_Int(int x) */ void PCU_Max_Longs(long* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); } long PCU_Max_Long(long x) @@ -509,43 +513,43 @@ int PCU_And(int c) */ int PCU_Proc_Self(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(global_mpi); + return pcu_mpi_rank(global_pcu.mpi); } /** \brief Returns the number of processes. */ int PCU_Proc_Peers(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(global_mpi); + return pcu_mpi_size(global_pcu.mpi); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ int PCU_Comm_Rank(int* rank) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(global_mpi); + *rank = pcu_mpi_rank(global_pcu.mpi); return PCU_SUCCESS; } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ int PCU_Comm_Size(int* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(global_mpi); + *size = pcu_mpi_size(global_pcu.mpi); return PCU_SUCCESS; } /** \brief Returns true iff PCU has been initialized */ bool PCU_Comm_Initialized(void) { - return global_state == init; + return global_pcu.state == pcu_state_init; } /** \brief Deprecated, see PCU_Comm_Begin. @@ -553,9 +557,9 @@ bool PCU_Comm_Initialized(void) int PCU_Comm_Start(PCU_Method method) { (void)method; //warning silencer - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(global_mpi,get_msg()); + pcu_msg_start(global_pcu.mpi,get_msg()); return PCU_SUCCESS; } @@ -566,9 +570,9 @@ int PCU_Comm_Start(PCU_Method method) */ int PCU_Comm_Packed(int to_rank, size_t* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) reel_fail("Invalid rank in Comm_Packed"); *size = pcu_msg_packed(get_msg(),to_rank); return PCU_SUCCESS; @@ -585,9 +589,9 @@ int PCU_Comm_Packed(int to_rank, size_t* size) */ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) reel_fail("Invalid rank in Comm_Write"); pcu_msg* msg = get_msg(); PCU_MSG_PACK(msg,to_rank,size); @@ -618,7 +622,7 @@ bool PCU_Comm_Receive(void) */ bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Read called before Comm_Init"); if (!PCU_Comm_Receive()) return false; @@ -649,7 +653,7 @@ static void append(char* s, size_t size, const char* format, ...) void PCU_Debug_Open(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Debug_Open called before Comm_Init"); const int fanout = 2048; @@ -677,7 +681,7 @@ void PCU_Debug_Open(void) /** \brief like fprintf, contents go to debugN.txt */ void PCU_Debug_Print(const char* format, ...) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Debug_Print called before Comm_Init"); pcu_msg* msg = get_msg(); if ( ! msg->file) @@ -692,7 +696,7 @@ void PCU_Debug_Print(const char* format, ...) /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ int PCU_Comm_From(int* from_rank) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_From called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -709,7 +713,7 @@ int PCU_Comm_From(int* from_rank) */ int PCU_Comm_Received(size_t* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Received called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -727,7 +731,7 @@ int PCU_Comm_Received(size_t* size) */ void* PCU_Comm_Extract(size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Extract called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -745,13 +749,13 @@ void* PCU_Comm_Extract(size_t size) */ void PCU_Switch_Comm(MPI_Comm new_comm) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Switch_Comm called before Comm_Init"); int result; - MPI_Comm_compare(new_comm, global_mpi->original_comm, &result); + MPI_Comm_compare(new_comm, global_pcu.mpi->original_comm, &result); if (result != MPI_IDENT) { - pcu_mpi_finalize(&global_mpi); - global_mpi = pcu_mpi_init(new_comm); + pcu_mpi_finalize(&(global_pcu.mpi)); + global_pcu.mpi = pcu_mpi_init(new_comm); } } @@ -762,9 +766,9 @@ void PCU_Switch_Comm(MPI_Comm new_comm) */ MPI_Comm PCU_Get_Comm(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Get_Comm called before Comm_Init"); - return global_mpi->original_comm; + return global_pcu.mpi->original_comm; } /** \brief Return the time in seconds since some time in the past From 4c243cbd5a0b662e2286efc28b140fa8a7aec3c9 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 03:07:50 -0400 Subject: [PATCH 005/141] Use consistent initialization functions for MPI. This commit allows the pcu_mpi to be initialized into a stack variable rather than allways allocating on the heap. --- pcu/pcu.c | 66 +++++++++++++++++++++++++------------------------- pcu/pcu_mpi.c | 6 ++--- pcu/pcu_mpi.h | 4 +-- pcu/pcu_pmpi.c | 27 ++++++--------------- pcu/pcu_pmpi.h | 4 +-- 5 files changed, 47 insertions(+), 60 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index 8dd48f0a2..a7f2c26fd 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -46,7 +46,7 @@ typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; typedef struct { pcu_state_enum state; pcu_msg pmsg; - pcu_mpi_t* mpi; + pcu_mpi_t mpi; } pcu_t; static pcu_t global_pcu = {.state=pcu_state_uninit}; @@ -65,7 +65,7 @@ int PCU_Comm_Init(void) { if (global_pcu.state != pcu_state_uninit) reel_fail("nested calls to Comm_Init"); - global_pcu.mpi = pcu_mpi_init(MPI_COMM_WORLD); + pcu_mpi_init(MPI_COMM_WORLD,&global_pcu.mpi); pcu_make_msg(&(global_pcu.pmsg)); global_pcu.state = pcu_state_init; /* turn ordering on by default, call @@ -104,7 +104,7 @@ int PCU_Comm_Self(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(global_pcu.mpi); + return pcu_mpi_rank(&global_pcu.mpi); } /** \brief Returns the number of threads in the program. @@ -115,7 +115,7 @@ int PCU_Comm_Peers(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(global_pcu.mpi); + return pcu_mpi_size(&global_pcu.mpi); } /** \brief Begins a PCU communication phase. @@ -128,7 +128,7 @@ void PCU_Comm_Begin(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(global_pcu.mpi, get_msg()); + pcu_msg_start(&global_pcu.mpi, get_msg()); } /** \brief Packs data to be sent to \a to_rank. @@ -141,7 +141,7 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) reel_fail("Invalid rank in Comm_Pack"); if ( size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); @@ -162,7 +162,7 @@ int PCU_Comm_Send(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(global_pcu.mpi, get_msg()); + pcu_msg_send(&global_pcu.mpi, get_msg()); return PCU_SUCCESS; } @@ -183,8 +183,8 @@ bool PCU_Comm_Listen(void) reel_fail("Comm_Listen called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) - return pcu_order_receive(global_pcu.mpi, m->order, m); - return pcu_msg_receive(global_pcu.mpi, m); + return pcu_order_receive(&global_pcu.mpi, m->order, m); + return pcu_msg_receive(&global_pcu.mpi, m); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -253,7 +253,7 @@ void PCU_Barrier(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Barrier called before Comm_Init"); - pcu_barrier(global_pcu.mpi, &(get_msg()->coll)); + pcu_barrier(&global_pcu.mpi, &(get_msg()->coll)); } /** \brief Performs an Allreduce sum of double arrays. @@ -266,7 +266,7 @@ void PCU_Add_Doubles(double* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); } double PCU_Add_Double(double x) @@ -283,7 +283,7 @@ void PCU_Min_Doubles(double* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); } double PCU_Min_Double(double x) @@ -300,7 +300,7 @@ void PCU_Max_Doubles(double* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); } double PCU_Max_Double(double x) @@ -317,7 +317,7 @@ void PCU_Add_Ints(int* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); } int PCU_Add_Int(int x) @@ -334,7 +334,7 @@ void PCU_Add_Longs(long* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); } long PCU_Add_Long(long x) @@ -351,7 +351,7 @@ void PCU_Add_SizeTs(size_t* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); } size_t PCU_Add_SizeT(size_t x) @@ -367,7 +367,7 @@ size_t PCU_Add_SizeT(size_t x) void PCU_Min_SizeTs(size_t* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); } size_t PCU_Min_SizeT(size_t x) { @@ -382,7 +382,7 @@ size_t PCU_Min_SizeT(size_t x) { void PCU_Max_SizeTs(size_t* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); } size_t PCU_Max_SizeT(size_t x) { @@ -406,7 +406,7 @@ void PCU_Exscan_Ints(int* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -430,7 +430,7 @@ void PCU_Exscan_Longs(long* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -451,7 +451,7 @@ void PCU_Min_Ints(int* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); } int PCU_Min_Int(int x) @@ -468,7 +468,7 @@ void PCU_Max_Ints(int* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); } int PCU_Max_Int(int x) @@ -484,7 +484,7 @@ void PCU_Max_Longs(long* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); } long PCU_Max_Long(long x) @@ -515,7 +515,7 @@ int PCU_Proc_Self(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(global_pcu.mpi); + return pcu_mpi_rank(&global_pcu.mpi); } /** \brief Returns the number of processes. @@ -524,7 +524,7 @@ int PCU_Proc_Peers(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(global_pcu.mpi); + return pcu_mpi_size(&global_pcu.mpi); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. @@ -533,7 +533,7 @@ int PCU_Comm_Rank(int* rank) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(global_pcu.mpi); + *rank = pcu_mpi_rank(&global_pcu.mpi); return PCU_SUCCESS; } @@ -542,7 +542,7 @@ int PCU_Comm_Size(int* size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(global_pcu.mpi); + *size = pcu_mpi_size(&global_pcu.mpi); return PCU_SUCCESS; } @@ -559,7 +559,7 @@ int PCU_Comm_Start(PCU_Method method) (void)method; //warning silencer if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(global_pcu.mpi,get_msg()); + pcu_msg_start(&global_pcu.mpi,get_msg()); return PCU_SUCCESS; } @@ -572,7 +572,7 @@ int PCU_Comm_Packed(int to_rank, size_t* size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) reel_fail("Invalid rank in Comm_Packed"); *size = pcu_msg_packed(get_msg(),to_rank); return PCU_SUCCESS; @@ -591,7 +591,7 @@ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) reel_fail("Invalid rank in Comm_Write"); pcu_msg* msg = get_msg(); PCU_MSG_PACK(msg,to_rank,size); @@ -752,10 +752,10 @@ void PCU_Switch_Comm(MPI_Comm new_comm) if (global_pcu.state == pcu_state_uninit) reel_fail("Switch_Comm called before Comm_Init"); int result; - MPI_Comm_compare(new_comm, global_pcu.mpi->original_comm, &result); + MPI_Comm_compare(new_comm, global_pcu.mpi.original_comm, &result); if (result != MPI_IDENT) { pcu_mpi_finalize(&(global_pcu.mpi)); - global_pcu.mpi = pcu_mpi_init(new_comm); + pcu_mpi_init(new_comm,&global_pcu.mpi); } } @@ -768,7 +768,7 @@ MPI_Comm PCU_Get_Comm(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Get_Comm called before Comm_Init"); - return global_pcu.mpi->original_comm; + return global_pcu.mpi.original_comm; } /** \brief Return the time in seconds since some time in the past diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index c5dea8baa..40236ef95 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -61,9 +61,9 @@ bool pcu_mpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) check_rank(self, m->peer); return pcu_pmpi_receive(self, m, comm); } -pcu_mpi_t* pcu_mpi_init(MPI_Comm comm) { - return pcu_pmpi_init(comm); +void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi) { + pcu_pmpi_init(comm, mpi); } -void pcu_mpi_finalize(pcu_mpi_t** mpi) { +void pcu_mpi_finalize(pcu_mpi_t* mpi) { pcu_pmpi_finalize(mpi); } diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 1bed6d6c7..9c0fad4f4 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -37,7 +37,7 @@ int pcu_mpi_rank(pcu_mpi_t*); void pcu_mpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); bool pcu_mpi_done(pcu_mpi_t*, pcu_message* m); bool pcu_mpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -pcu_mpi_t* pcu_mpi_init(MPI_Comm comm); -void pcu_mpi_finalize(pcu_mpi_t**); +void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); +void pcu_mpi_finalize(pcu_mpi_t* mpi); #endif diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index e8f546d70..a4280dace 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -15,28 +15,9 @@ void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); -void pcu_pmpi_destruct(pcu_mpi_t* self); -void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm); - -pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) -{ - pcu_mpi_t* m = (pcu_mpi_t*)malloc(sizeof(pcu_mpi_t)); - pcu_pmpi_construct(m,comm); - return m; -} - -void pcu_pmpi_finalize(pcu_mpi_t** m) +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* self) { - pcu_pmpi_destruct(*m); - free(*m); - *m = NULL; -} -void pcu_pmpi_destruct(pcu_mpi_t* self) { - MPI_Comm_free(&(self->user_comm)); - MPI_Comm_free(&(self->coll_comm)); -} -void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { self->original_comm = comm; MPI_Comm_dup(comm,&(self->user_comm)); MPI_Comm_dup(comm,&(self->coll_comm)); @@ -44,6 +25,12 @@ void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { MPI_Comm_rank(comm,&(self->rank)); } +void pcu_pmpi_finalize(pcu_mpi_t* self) +{ + MPI_Comm_free(&(self->user_comm)); + MPI_Comm_free(&(self->coll_comm)); +} + int pcu_pmpi_size(pcu_mpi_t* self) { return self->size; diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index cdfbbf699..36507421d 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -14,8 +14,8 @@ #include -pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm); -void pcu_pmpi_finalize(pcu_mpi_t** m); +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* mpi); +void pcu_pmpi_finalize(pcu_mpi_t* m); int pcu_pmpi_size(pcu_mpi_t* self); int pcu_pmpi_rank(pcu_mpi_t* self); void pcu_pmpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); From d891d2b377e5e58976913973de0fe9f914d7453b Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 14:08:36 -0400 Subject: [PATCH 006/141] Add PCU2 interface that takes the PCU state struct --- pcu/CMakeLists.txt | 1 + pcu/PCU2.h | 137 ++++++++ pcu/pcu.c | 392 ++++------------------- pcu/pcu2.c | 783 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 985 insertions(+), 328 deletions(-) create mode 100644 pcu/PCU2.h create mode 100644 pcu/pcu2.c diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 91649e43f..9635d0d40 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -10,6 +10,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES pcu.c + pcu2.c pcu_aa.c pcu_coll.c pcu_io.c diff --git a/pcu/PCU2.h b/pcu/PCU2.h new file mode 100644 index 000000000..3601ff029 --- /dev/null +++ b/pcu/PCU2.h @@ -0,0 +1,137 @@ +/****************************************************************************** + + Copyright 2011 Scientific Computation Research Center, + Rensselaer Polytechnic Institute. All rights reserved. + + This work is open source software, licensed under the terms of the + BSD license as described in the LICENSE file in the top-level directory. + +*******************************************************************************/ +#ifndef PCU2_H +#define PCU2_H + +#define PCU_SUCCESS 0 +#define PCU_FAILURE -1 + +#include + +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif + +// FOR pcu_state_type ... use opaque ptr instead? +#include "pcu_msg.h" + +typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; +typedef struct { + pcu_state_enum state; + pcu_msg pmsg; + pcu_mpi_t mpi; +} pcu_t; + +/*library init/finalize*/ +int PCU_Comm_Init_2(pcu_t* pcu); +int PCU_Comm_Free_2(pcu_t* pcu); + +/*rank/size functions*/ +int PCU_Comm_Self_2(pcu_t* pcu); +int PCU_Comm_Peers_2(pcu_t* pcu); + +/*recommended message passing API*/ +void PCU_Comm_Begin_2(pcu_t* pcu); +int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size); +int PCU_Comm_Send_2(pcu_t* pcu); +bool PCU_Comm_Receive_2(pcu_t* pcu); +bool PCU_Comm_Listen_2(pcu_t* pcu); +int PCU_Comm_Sender_2(pcu_t* pcu); +bool PCU_Comm_Unpacked_2(pcu_t* pcu); +int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size); + +/*turns deterministic ordering for the + above API on/off*/ +void PCU_Comm_Order_2(pcu_t* pcu, bool on); + +/*collective operations*/ +void PCU_Barrier_2(pcu_t* pcu); +void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n); +double PCU_Add_Double_2(pcu_t* pcu, double x); +void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n); +double PCU_Min_Double_2(pcu_t* pcu, double x); +void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n); +double PCU_Max_Double_2(pcu_t* pcu, double x); +void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Add_Int_2(pcu_t* pcu, int x); +void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n); +long PCU_Add_Long_2(pcu_t* pcu, long x); +void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Exscan_Int_2(pcu_t* pcu, int x); +void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n); +long PCU_Exscan_Long_2(pcu_t* pcu, long x); +void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); +size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x); +void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); +size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x); +void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); +size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x); +void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Min_Int_2(pcu_t* pcu, int x); +void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Max_Int_2(pcu_t* pcu, int x); +void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n); +long PCU_Max_Long_2(pcu_t* pcu, long x); +int PCU_Or_2(pcu_t* pcu, int c); +int PCU_And_2(pcu_t* pcu, int c); + +/*process-level self/peers (mpi wrappers)*/ +int PCU_Proc_Self_2(pcu_t* pcu); +int PCU_Proc_Peers_2(pcu_t* pcu); + +/*IPComMan replacement API*/ +int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size); +bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size); + +/*Debug file I/O API*/ +void PCU_Debug_Open_2(pcu_t* pcu); +void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list args); +#ifdef __GNUC__ +void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) + __attribute__((format(printf,2,3))); +#else +void PCU_Debug_Print(pcu_t* pcu, const char* format, ...); +#endif + +/*lesser-used APIs*/ +bool PCU_Comm_Initialized_2(pcu_t* pcu); +int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size); +int PCU_Comm_From_2(pcu_t* pcu, int* from_rank); +int PCU_Comm_Received_2(pcu_t* pcu, size_t* size); +void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size); +int PCU_Comm_Rank_2(pcu_t* pcu, int* rank); +int PCU_Comm_Size_2(pcu_t* pcu, int* size); + +int PCU_Comm_Start_2(pcu_t* pcu); + +/*special MPI_Comm replacement API*/ +void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm); +MPI_Comm PCU_Get_Comm_2(pcu_t* pcu); + +/*stack trace helpers using GNU/Linux*/ +void PCU_Protect_2(pcu_t* pcu); + +/*MPI_Wtime_() equivalent*/ +double PCU_Time_2(pcu_t* pcu); + +/*Memory usage*/ +double PCU_GetMem_2(pcu_t* pcu); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/pcu/pcu.c b/pcu/pcu.c index a7f2c26fd..d4679fc86 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -32,6 +32,7 @@ #include #include #include "PCU.h" +#include "PCU2.h" #include "pcu_msg.h" #include "pcu_order.h" #include "noto_malloc.h" @@ -42,20 +43,9 @@ #include /*INT_MAX*/ #include /*abort*/ -typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; -typedef struct { - pcu_state_enum state; - pcu_msg pmsg; - pcu_mpi_t mpi; -} pcu_t; static pcu_t global_pcu = {.state=pcu_state_uninit}; -static pcu_msg* get_msg() -{ - return &(global_pcu.pmsg); -} - /** \brief Initializes the PCU library. \details This function must be called by all MPI processes before calling any other PCU functions. @@ -63,16 +53,7 @@ static pcu_msg* get_msg() */ int PCU_Comm_Init(void) { - if (global_pcu.state != pcu_state_uninit) - reel_fail("nested calls to Comm_Init"); - pcu_mpi_init(MPI_COMM_WORLD,&global_pcu.mpi); - pcu_make_msg(&(global_pcu.pmsg)); - global_pcu.state = pcu_state_init; - /* turn ordering on by default, call - PCU_Comm_Order(false) after PCU_Comm_Init - to disable this */ - PCU_Comm_Order(true); - return PCU_SUCCESS; + PCU_Comm_Init_2(&global_pcu); } /** \brief Frees all PCU library structures. @@ -81,14 +62,7 @@ int PCU_Comm_Init(void) */ int PCU_Comm_Free(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Free called before Comm_Init"); - if (global_pcu.pmsg.order) - pcu_order_free(global_pcu.pmsg.order); - pcu_free_msg(&global_pcu.pmsg); - pcu_mpi_finalize(&global_pcu.mpi); - global_pcu.state = pcu_state_uninit; - return PCU_SUCCESS; + return PCU_Comm_Free_2(&global_pcu); } /** \brief Returns the communication rank of the calling thread. @@ -102,9 +76,7 @@ int PCU_Comm_Free(void) */ int PCU_Comm_Self(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(&global_pcu.mpi); + return PCU_Comm_Self_2(&global_pcu); } /** \brief Returns the number of threads in the program. @@ -113,9 +85,7 @@ int PCU_Comm_Self(void) */ int PCU_Comm_Peers(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(&global_pcu.mpi); + return PCU_Comm_Peers_2(&global_pcu); } /** \brief Begins a PCU communication phase. @@ -126,9 +96,7 @@ int PCU_Comm_Peers(void) */ void PCU_Comm_Begin(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(&global_pcu.mpi, get_msg()); + PCU_Comm_Begin_2(&global_pcu); } /** \brief Packs data to be sent to \a to_rank. @@ -139,16 +107,7 @@ void PCU_Comm_Begin(void) */ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) - reel_fail("Invalid rank in Comm_Pack"); - if ( size > (size_t)INT_MAX ) { - fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); - abort(); - } - memcpy(pcu_msg_pack(get_msg(),to_rank,size),data,size); - return PCU_SUCCESS; + return PCU_Comm_Pack_2(&global_pcu, to_rank, data, size); } /** \brief Sends all buffers for this communication phase. @@ -160,10 +119,7 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) */ int PCU_Comm_Send(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(&global_pcu.mpi, get_msg()); - return PCU_SUCCESS; + return PCU_Comm_Send_2(&global_pcu); } /** \brief Tries to receive a buffer for this communication phase. @@ -179,12 +135,7 @@ int PCU_Comm_Send(void) */ bool PCU_Comm_Listen(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Listen called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_receive(&global_pcu.mpi, m->order, m); - return pcu_msg_receive(&global_pcu.mpi, m); + return PCU_Comm_Listen_2(&global_pcu); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -192,12 +143,7 @@ bool PCU_Comm_Listen(void) */ int PCU_Comm_Sender(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Sender called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_received_from(m->order); - return pcu_msg_received_from(m); + return PCU_Comm_Sender_2(&global_pcu); } /** \brief Returns true if the current received buffer has been unpacked. @@ -205,12 +151,7 @@ int PCU_Comm_Sender(void) */ bool PCU_Comm_Unpacked(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Unpacked called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_unpacked(m->order); - return pcu_msg_unpacked(m); + return PCU_Comm_Unpacked_2(&global_pcu); } /** \brief Unpacks a block of data from the current received buffer. @@ -225,35 +166,18 @@ bool PCU_Comm_Unpacked(void) */ int PCU_Comm_Unpack(void* data, size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Unpack called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - memcpy(data,pcu_order_unpack(m->order,size),size); - else - memcpy(data,pcu_msg_unpack(m,size),size); - return PCU_SUCCESS; + return PCU_Comm_Unpack_2(&global_pcu, data, size); } void PCU_Comm_Order(bool on) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Order called before Comm_Init"); - pcu_msg* m = get_msg(); - if (on && (!m->order)) - m->order = pcu_order_new(); - if ((!on) && m->order) { - pcu_order_free(m->order); - m->order = NULL; - } + PCU_Comm_Order_2(&global_pcu, on); } /** \brief Blocking barrier over all threads. */ void PCU_Barrier(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&global_pcu.mpi, &(get_msg()->coll)); + PCU_Barrier_2(&global_pcu); } /** \brief Performs an Allreduce sum of double arrays. @@ -264,132 +188,92 @@ void PCU_Barrier(void) */ void PCU_Add_Doubles(double* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + PCU_Add_Doubles_2(&global_pcu, p, n); } double PCU_Add_Double(double x) { - double a[1]; - a[0] = x; - PCU_Add_Doubles(a, 1); - return a[0]; + return PCU_Add_Double_2(&global_pcu, x); } /** \brief Performs an Allreduce minimum of double arrays. */ void PCU_Min_Doubles(double* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + PCU_Min_Doubles_2(&global_pcu, p, n); } double PCU_Min_Double(double x) { - double a[1]; - a[0] = x; - PCU_Min_Doubles(a, 1); - return a[0]; + return PCU_Min_Double_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of double arrays. */ void PCU_Max_Doubles(double* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + PCU_Max_Doubles_2(&global_pcu, p, n); } double PCU_Max_Double(double x) { - double a[1]; - a[0] = x; - PCU_Max_Doubles(a, 1); - return a[0]; + return PCU_Max_Double_2(&global_pcu, x); } /** \brief Performs an Allreduce sum of integers */ void PCU_Add_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + PCU_Add_Ints_2(&global_pcu, p, n); } int PCU_Add_Int(int x) { - int a[1]; - a[0] = x; - PCU_Add_Ints(a, 1); - return a[0]; + return PCU_Add_Int_2(&global_pcu, x); } /** \brief Performs an Allreduce sum of long integers */ void PCU_Add_Longs(long* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + PCU_Add_Longs_2(&global_pcu, p, n); } long PCU_Add_Long(long x) { - long a[1]; - a[0] = x; - PCU_Add_Longs(a, 1); - return a[0]; + return PCU_Add_Long_2(&global_pcu, x); } /** \brief Performs an Allreduce sum of size_t unsigned integers */ void PCU_Add_SizeTs(size_t* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + PCU_Add_SizeTs_2(&global_pcu, p, n); } size_t PCU_Add_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Add_SizeTs(a, 1); - return a[0]; + return PCU_Add_SizeT_2(&global_pcu, x); } /** \brief Performs an Allreduce minimum of size_t unsigned integers */ void PCU_Min_SizeTs(size_t* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + PCU_Min_SizeTs_2(&global_pcu, p, n); } size_t PCU_Min_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Min_SizeTs(a, 1); - return a[0]; + return PCU_Min_SizeT_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of size_t unsigned integers */ void PCU_Max_SizeTs(size_t* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + PCU_Max_SizeTs_2(&global_pcu, p, n); } size_t PCU_Max_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Max_SizeTs(a, 1); - return a[0]; + return PCU_Max_SizeT_2(&global_pcu, x); } /** \brief Performs an exclusive prefix sum of integer arrays. @@ -400,156 +284,105 @@ size_t PCU_Max_SizeT(size_t x) { */ void PCU_Exscan_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Exscan_Ints called before Comm_Init"); - int* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); + PCU_Exscan_Ints_2(&global_pcu, p, n); } int PCU_Exscan_Int(int x) { - int a[1]; - a[0] = x; - PCU_Exscan_Ints(a, 1); - return a[0]; + return PCU_Exscan_Int_2(&global_pcu, x); } /** \brief See PCU_Exscan_Ints */ void PCU_Exscan_Longs(long* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Exscan_Longs called before Comm_Init"); - long* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); + PCU_Exscan_Longs_2(&global_pcu, p, n); } long PCU_Exscan_Long(long x) { - long a[1]; - a[0] = x; - PCU_Exscan_Longs(a, 1); - return a[0]; + return PCU_Exscan_Long_2(&global_pcu, x); } /** \brief Performs an Allreduce minimum of int arrays. */ void PCU_Min_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + PCU_Min_Ints_2(&global_pcu, p, n); } int PCU_Min_Int(int x) { - int a[1]; - a[0] = x; - PCU_Min_Ints(a, 1); - return a[0]; + return PCU_Min_Int_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of int arrays. */ void PCU_Max_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + PCU_Max_Ints_2(&global_pcu, p, n); } int PCU_Max_Int(int x) { - int a[1]; - a[0] = x; - PCU_Max_Ints(a, 1); - return a[0]; + return PCU_Max_Int_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of long arrays. */ void PCU_Max_Longs(long* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + PCU_Max_Longs_2(&global_pcu, p, n); } long PCU_Max_Long(long x) { - long a[1]; - a[0] = x; - PCU_Max_Longs(a, 1); - return a[0]; + return PCU_Max_Long_2(&global_pcu, x); } /** \brief Performs a parallel logical OR reduction */ int PCU_Or(int c) { - return PCU_Max_Int(c); + return PCU_Or_2(&global_pcu, c); } /** \brief Performs a parallel logical AND reduction */ int PCU_And(int c) { - return PCU_Min_Int(c); + return PCU_And_2(&global_pcu, c); } /** \brief Returns the unique rank of the calling process. */ int PCU_Proc_Self(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(&global_pcu.mpi); + return PCU_Proc_Self_2(&global_pcu); } /** \brief Returns the number of processes. */ int PCU_Proc_Peers(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(&global_pcu.mpi); + return PCU_Proc_Peers_2(&global_pcu); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ int PCU_Comm_Rank(int* rank) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(&global_pcu.mpi); - return PCU_SUCCESS; + return PCU_Comm_Rank_2(&global_pcu, rank); } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ int PCU_Comm_Size(int* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(&global_pcu.mpi); - return PCU_SUCCESS; + return PCU_Comm_Size_2(&global_pcu, size); } /** \brief Returns true iff PCU has been initialized */ bool PCU_Comm_Initialized(void) { - return global_pcu.state == pcu_state_init; + return PCU_Comm_Initialized_2(&global_pcu); } /** \brief Deprecated, see PCU_Comm_Begin. @@ -557,10 +390,7 @@ bool PCU_Comm_Initialized(void) int PCU_Comm_Start(PCU_Method method) { (void)method; //warning silencer - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(&global_pcu.mpi,get_msg()); - return PCU_SUCCESS; + return PCU_Comm_Start_2(&global_pcu); } /** \brief Returns in * \a size the number of bytes being sent to \a to_rank. @@ -570,12 +400,7 @@ int PCU_Comm_Start(PCU_Method method) */ int PCU_Comm_Packed(int to_rank, size_t* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) - reel_fail("Invalid rank in Comm_Packed"); - *size = pcu_msg_packed(get_msg(),to_rank); - return PCU_SUCCESS; + return PCU_Comm_Packed_2(&global_pcu, to_rank, size); } /** \brief Packs a message to be sent to \a to_rank. @@ -589,23 +414,13 @@ int PCU_Comm_Packed(int to_rank, size_t* size) */ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) - reel_fail("Invalid rank in Comm_Write"); - pcu_msg* msg = get_msg(); - PCU_MSG_PACK(msg,to_rank,size); - memcpy(pcu_msg_pack(msg,to_rank,size),data,size); - return PCU_SUCCESS; + return PCU_Comm_Write_2(&global_pcu, to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ bool PCU_Comm_Receive(void) { - while (PCU_Comm_Unpacked()) - if (!PCU_Comm_Listen()) - return false; - return true; + return PCU_Comm_Receive_2(&global_pcu); } /** \brief Receives a message for this communication phase. @@ -622,88 +437,30 @@ bool PCU_Comm_Receive(void) */ bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Read called before Comm_Init"); - if (!PCU_Comm_Receive()) - return false; - *from_rank = PCU_Comm_Sender(); - PCU_COMM_UNPACK(*size); - *data = PCU_Comm_Extract(*size); - return true; + return PCU_Comm_Read_2(&global_pcu, from_rank, data, size); } -static void safe_mkdir(const char* path, mode_t mode) -{ - int err; - errno = 0; - err = mkdir(path, mode); - if (err != 0 && errno != EEXIST) - reel_fail("PCU: could not create directory \"%s\"\n", path); -} -static void append(char* s, size_t size, const char* format, ...) -{ - int len = strlen(s); - va_list ap; - va_start(ap, format); - vsnprintf(s + len, size - len, format, ap); - va_end(ap); -} void PCU_Debug_Open(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Debug_Open called before Comm_Init"); - - const int fanout = 2048; - const int bufsize = 1024; - char* path = noto_malloc(bufsize); - path[0] = '\0'; - if (PCU_Comm_Peers() > fanout) { - mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - strcpy(path, "debug/"); - safe_mkdir(path, dir_perm); - int self = PCU_Comm_Self(); - append(path, bufsize, "%d/", self / fanout); - if (self % fanout == 0) - safe_mkdir(path, dir_perm); - PCU_Barrier(); - } - - append(path,bufsize, "%s", "debug"); - pcu_msg* msg = get_msg(); - if ( ! msg->file) - msg->file = pcu_open_parallel(path,"txt"); - noto_free(path); + PCU_Debug_Open_2(&global_pcu); } /** \brief like fprintf, contents go to debugN.txt */ void PCU_Debug_Print(const char* format, ...) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Debug_Print called before Comm_Init"); - pcu_msg* msg = get_msg(); - if ( ! msg->file) - return; //Print is a no-op if no file is open - va_list ap; - va_start(ap,format); - vfprintf(msg->file,format,ap); - va_end(ap); - fflush(msg->file); + va_list arglist; + va_start(arglist, format); + PCU_Debug_Printv_2(&global_pcu, format, arglist); + va_end(arglist); } /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ int PCU_Comm_From(int* from_rank) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_From called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - *from_rank = pcu_order_received_from(m->order); - else - *from_rank = pcu_msg_received_from(m); - return PCU_SUCCESS; + return PCU_Comm_From_2(&global_pcu, from_rank); } /** \brief Returns in * \a size the bytes in the current received buffer @@ -713,14 +470,7 @@ int PCU_Comm_From(int* from_rank) */ int PCU_Comm_Received(size_t* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Received called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - *size = pcu_order_received_size(m->order); - else - *size = pcu_msg_received_size(m); - return PCU_SUCCESS; + return PCU_Comm_Received_2(&global_pcu, size); } /** \brief Extracts a block of data from the current received buffer. @@ -731,12 +481,7 @@ int PCU_Comm_Received(size_t* size) */ void* PCU_Comm_Extract(size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Extract called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_unpack(m->order,size); - return pcu_msg_unpack(m,size); + return PCU_Comm_Extract_2(&global_pcu, size); } /** \brief Reinitializes PCU with a new MPI communicator. @@ -749,14 +494,7 @@ void* PCU_Comm_Extract(size_t size) */ void PCU_Switch_Comm(MPI_Comm new_comm) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Switch_Comm called before Comm_Init"); - int result; - MPI_Comm_compare(new_comm, global_pcu.mpi.original_comm, &result); - if (result != MPI_IDENT) { - pcu_mpi_finalize(&(global_pcu.mpi)); - pcu_mpi_init(new_comm,&global_pcu.mpi); - } + PCU_Switch_Comm_2(&global_pcu, new_comm); } /** \brief Return the current MPI communicator @@ -766,19 +504,17 @@ void PCU_Switch_Comm(MPI_Comm new_comm) */ MPI_Comm PCU_Get_Comm(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Get_Comm called before Comm_Init"); - return global_pcu.mpi.original_comm; + PCU_Get_Comm_2(&global_pcu); } /** \brief Return the time in seconds since some time in the past */ double PCU_Time(void) { - return MPI_Wtime(); + return PCU_Time_2(&global_pcu); } void PCU_Protect(void) { - reel_protect(); + PCU_Protect_2(&global_pcu); } diff --git a/pcu/pcu2.c b/pcu/pcu2.c new file mode 100644 index 000000000..001ac9ed2 --- /dev/null +++ b/pcu/pcu2.c @@ -0,0 +1,783 @@ +/****************************************************************************** + + Copyright 2011 Scientific Computation Research Center, + Rensselaer Polytechnic Institute. All rights reserved. + + This work is open source software, licensed under the terms of the + BSD license as described in the LICENSE file in the top-level directory. + +*******************************************************************************/ +/** \file pcu->c + \brief The PCU communication interface */ +/** \page pcu PCU + PCU (the Parallel Control Utility) is a library for parallel computation + based on MPI. + PCU provides three things to users: + 1. A hybrid phased message passing system + 2. Hybrid collective operations + + Phased message passing is similar to Bulk Synchronous Parallel. + All messages are exchanged in a phase, which is a collective operation + involving all threads in the parallel program. + During a phase, the following events happen in sequence: + 1. All threads send non-blocking messages to other threads + 2. All threads receive all messages sent to them during this phase + PCU provides termination detection, which is the ability to detect when all + messages have been received without prior knowledge of which threads + are sending to which. + + The API documentation is here: pcu->c +*/ + +#include +#include +#include "PCU2.h" +#include "pcu_msg.h" +#include "pcu_order.h" +#include "noto_malloc.h" +#include "reel.h" +#include /*required for mode_t for mkdir on some systems*/ +#include /*using POSIX mkdir call for SMB "foo/" path*/ +#include /* for checking the error from mkdir */ +#include /*INT_MAX*/ +#include /*abort*/ + + +static pcu_msg* get_msg(pcu_t * pcu) +{ + return &(pcu->pmsg); +} + +/** \brief Initializes the PCU library. + \details This function must be called by all MPI processes before + calling any other PCU functions. + MPI_Init or MPI_Init_thread should be called before this function. + */ +int PCU_Comm_Init_2(pcu_t* pcu) +{ + if (pcu->state != pcu_state_uninit) + reel_fail("nested calls to Comm_Init"); + pcu_mpi_init(MPI_COMM_WORLD,&(pcu->mpi)); + pcu_make_msg(&(pcu->pmsg)); + pcu->state = pcu_state_init; + /* turn ordering on by default, call + PCU_Comm_Order(false) after PCU_Comm_Init + to disable this */ + PCU_Comm_Order_2(pcu, true); + return PCU_SUCCESS; +} + +/** \brief Frees all PCU library structures. + \details This function must be called by all MPI processes after all + other calls to PCU, and before calling MPI_Finalize. + */ +int PCU_Comm_Free_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Free called before Comm_Init"); + if (pcu->pmsg.order) + pcu_order_free(pcu->pmsg.order); + pcu_free_msg(&(pcu->pmsg)); + pcu_mpi_finalize(&(pcu->mpi)); + pcu->state = pcu_state_uninit; + return PCU_SUCCESS; +} + +/** \brief Returns the communication rank of the calling thread. + \details when called from a non-threaded MPI process, this function is + equivalent to MPI_Comm_rank(MPI_COMM_WORLD,rank). + + Ranks are consecutive from 0 to \f$pt-1\f$ for a program with + \f$p\f$ processes and \f$t\f$ threads per process. + Ranks are contiguous within a process, so that the \f$t\f$ threads in process + \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. + */ +int PCU_Comm_Self_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Self called before Comm_Init"); + return pcu_mpi_rank(&(pcu->mpi)); +} + +/** \brief Returns the number of threads in the program. + \details when called from a non-threaded MPI process, this function is + equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). + */ +int PCU_Comm_Peers_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Peers called before Comm_Init"); + return pcu_mpi_size(&(pcu->mpi)); +} + +/** \brief Begins a PCU communication phase. + \details This function must be called by all threads in the MPI job + at the beginning of each phase of communication. + After calling this function, each thread may call functions like + PCU_Comm_Pack or PCU_Comm_Write. +*/ +void PCU_Comm_Begin_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Begin called before Comm_Init"); + pcu_msg_start(&(pcu->mpi), get_msg(pcu)); +} + +/** \brief Packs data to be sent to \a to_rank. + \details This function appends the block of \a size bytes starting + at \a data to the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Pack called before Comm_Init"); + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) + reel_fail("Invalid rank in Comm_Pack"); + if ( size > (size_t)INT_MAX ) { + fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); + abort(); + } + memcpy(pcu_msg_pack(get_msg(pcu),to_rank,size),data,size); + return PCU_SUCCESS; +} + +/** \brief Sends all buffers for this communication phase. + \details This function should be called by all threads in the MPI job + after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls + to PCU_Comm_Listen or PCU_Comm_Read. + All buffers from this thread are sent out and receiving + may begin after this call. + */ +int PCU_Comm_Send_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Send called before Comm_Init"); + pcu_msg_send(&(pcu->mpi), get_msg(pcu)); + return PCU_SUCCESS; +} + +/** \brief Tries to receive a buffer for this communication phase. + \details Either this function or PCU_Comm_Read should be called at least + once by all threads during the communication phase, after PCU_Comm_Send + is called. The result will be false if and only if the communication phase + is over and there are no more buffers to receive. + Otherwise, a buffer was received. + Its contents are retrievable through PCU_Comm_Unpack, and its metadata through + PCU_Comm_Sender and PCU_Comm_Received. + Users should unpack all data from this buffer before calling this function + again, because the previously received buffer is destroyed by the call. + */ +bool PCU_Comm_Listen_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Listen called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_receive(&(pcu->mpi), m->order, m); + return pcu_msg_receive(&(pcu->mpi), m); +} + +/** \brief Returns in * \a from_rank the sender of the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + */ +int PCU_Comm_Sender_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Sender called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_received_from(m->order); + return pcu_msg_received_from(m); +} + +/** \brief Returns true if the current received buffer has been unpacked. + \details This function should be called after a successful PCU_Comm_Listen. + */ +bool PCU_Comm_Unpacked_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Unpacked called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_unpacked(m->order); + return pcu_msg_unpacked(m); +} + +/** \brief Unpacks a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + \a data must point to a block of memory of at least \a size bytes, into + which the next \a size bytes of the current received buffer will be written. + Subsequent calls will begin unpacking where this call left off, + so that the entire received buffer can be unpacked by a sequence of calls to + this function. + Users must ensure that there remain \a size bytes to be unpacked, + PCU_Comm_Unpacked can help with this. + */ +int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Unpack called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + memcpy(data,pcu_order_unpack(m->order,size),size); + else + memcpy(data,pcu_msg_unpack(m,size),size); + return PCU_SUCCESS; +} + +void PCU_Comm_Order_2(pcu_t* pcu, bool on) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Order called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (on && (!m->order)) + m->order = pcu_order_new(); + if ((!on) && m->order) { + pcu_order_free(m->order); + m->order = NULL; + } +} + +/** \brief Blocking barrier over all threads. */ +void PCU_Barrier_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Barrier called before Comm_Init"); + pcu_barrier(&(pcu->mpi), &(get_msg(pcu)->coll)); +} + +/** \brief Performs an Allreduce sum of double arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n doubles. + After this call, p[i] will contain the sum of all p[i]'s + given by each rank. + */ +void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_Doubles called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_doubles,p,n*sizeof(double)); +} + +double PCU_Add_Double_2(pcu_t* pcu, double x) +{ + double a[1]; + a[0] = x; + PCU_Add_Doubles_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce minimum of double arrays. + */ +void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Min_Doubles called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_doubles,p,n*sizeof(double)); +} + +double PCU_Min_Double_2(pcu_t* pcu, double x) +{ + double a[1]; + a[0] = x; + PCU_Min_Doubles_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce maximum of double arrays. + */ +void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_Doubles called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_doubles,p,n*sizeof(double)); +} + +double PCU_Max_Double_2(pcu_t* pcu, double x) +{ + double a[1]; + a[0] = x; + PCU_Max_Doubles_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce sum of integers + */ +void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_Ints called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); +} + +int PCU_Add_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Add_Ints_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce sum of long integers + */ +void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_Longs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); +} + +long PCU_Add_Long_2(pcu_t* pcu, long x) +{ + long a[1]; + a[0] = x; + PCU_Add_Longs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce sum of size_t unsigned integers + */ +void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_SizeTs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_sizets,p,n*sizeof(size_t)); +} + +size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x) +{ + size_t a[1]; + a[0] = x; + PCU_Add_SizeTs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce minimum of size_t unsigned integers + */ +void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Min_SizeTs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_sizets,p,n*sizeof(size_t)); +} + +size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x) +{ + size_t a[1]; + a[0] = x; + PCU_Min_SizeTs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce maximum of size_t unsigned integers + */ +void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_SizeTs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_sizets,p,n*sizeof(size_t)); +} + +size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x) +{ + size_t a[1]; + a[0] = x; + PCU_Max_SizeTs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an exclusive prefix sum of integer arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n integers. + After this call, p[i] will contain the sum of all p[i]'s + given by ranks lower than the calling rank. + */ +void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Exscan_Ints called before Comm_Init"); + int* originals; + NOTO_MALLOC(originals,n); + for (size_t i=0; i < n; ++i) + originals[i] = p[i]; + pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); + //convert inclusive scan to exclusive + for (size_t i=0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} + +int PCU_Exscan_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Exscan_Ints_2(pcu, a, 1); + return a[0]; +} + +/** \brief See PCU_Exscan_Ints */ +void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Exscan_Longs called before Comm_Init"); + long* originals; + NOTO_MALLOC(originals,n); + for (size_t i=0; i < n; ++i) + originals[i] = p[i]; + pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); + //convert inclusive scan to exclusive + for (size_t i=0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} + +long PCU_Exscan_Long_2(pcu_t* pcu, long x) +{ + long a[1]; + a[0] = x; + PCU_Exscan_Longs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce minimum of int arrays. + */ +void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Min_Ints called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_ints,p,n*sizeof(int)); +} + +int PCU_Min_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Min_Ints_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce maximum of int arrays. + */ +void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_Ints called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_ints,p,n*sizeof(int)); +} + +int PCU_Max_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Max_Ints_2(pcu, a, 1); + return a[0]; +} +/** \brief Performs an Allreduce maximum of long arrays. + */ +void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_Longs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_longs,p,n*sizeof(long)); +} + +long PCU_Max_Long_2(pcu_t* pcu, long x) +{ + long a[1]; + a[0] = x; + PCU_Max_Longs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs a parallel logical OR reduction + */ +int PCU_Or_2(pcu_t* pcu, int c) +{ + return PCU_Max_Int_2(pcu, c); +} + +/** \brief Performs a parallel logical AND reduction + */ +int PCU_And_2(pcu_t* pcu, int c) +{ + return PCU_Min_Int_2(pcu, c); +} + +/** \brief Returns the unique rank of the calling process. + */ +int PCU_Proc_Self_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Proc_Self called before Comm_Init"); + return pcu_mpi_rank(&(pcu->mpi)); +} + +/** \brief Returns the number of processes. + */ +int PCU_Proc_Peers_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Proc_Peers called before Comm_Init"); + return pcu_mpi_size(&(pcu->mpi)); +} + +/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. + */ +int PCU_Comm_Rank_2(pcu_t* pcu, int* rank) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Rank called before Comm_Init"); + *rank = pcu_mpi_rank(&(pcu->mpi)); + return PCU_SUCCESS; +} + +/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ +int PCU_Comm_Size_2(pcu_t* pcu, int* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Size called before Comm_Init"); + *size = pcu_mpi_size(&(pcu->mpi)); + return PCU_SUCCESS; +} + +/** \brief Returns true iff PCU has been initialized */ +bool PCU_Comm_Initialized_2(pcu_t* pcu) +{ + return pcu->state == pcu_state_init; +} + +/** \brief Deprecated, see PCU_Comm_Begin. + */ +int PCU_Comm_Start_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Start called before Comm_Init"); + pcu_msg_start(&(pcu->mpi),get_msg(pcu)); + return PCU_SUCCESS; +} + +/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. + \details Returns the size of the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Packed called before Comm_Init"); + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) + reel_fail("Invalid rank in Comm_Packed"); + *size = pcu_msg_packed(get_msg(pcu),to_rank); + return PCU_SUCCESS; +} + +/** \brief Packs a message to be sent to \a to_rank. + \details This function packs a message into the buffer being sent + to \a to_rank. + Messages packed by this function can be received using the function + PCU_Comm_Read. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + If this function is used, PCU_Comm_Pack should not be used. + */ +int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Write called before Comm_Init"); + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) + reel_fail("Invalid rank in Comm_Write"); + pcu_msg* msg = get_msg(pcu); + PCU_MSG_PACK(msg,to_rank,size); + memcpy(pcu_msg_pack(msg,to_rank,size),data,size); + return PCU_SUCCESS; +} + +/** \brief Convenience wrapper over Listen and Unpacked */ +bool PCU_Comm_Receive_2(pcu_t* pcu) +{ + while (PCU_Comm_Unpacked_2(pcu)) + if (!PCU_Comm_Listen_2(pcu)) + return false; + return true; +} + +/** \brief Receives a message for this communication phase. + \details This function tries to receive a message packed by + PCU_Comm_Write. + If a the communication phase is over and there are no more + messages to receive, this function returns false. + Otherwise, * \a from_rank will be the rank which sent the message, + * \a data will point to the start of the message data, and + * \a size will be the number of bytes of message data. + If this function is used, PCU_Comm_Receive should not be used. + Note that the address * \a data points into a PCU buffer, so + it is strongly recommended that this data be read and not modified. + */ +bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Read called before Comm_Init"); + if (!PCU_Comm_Receive_2(pcu)) + return false; + *from_rank = PCU_Comm_Sender_2(pcu); + PCU_Comm_Unpack_2(pcu, size, sizeof(*size)); + *data = PCU_Comm_Extract_2(pcu, *size); + return true; +} + +static void safe_mkdir(const char* path, mode_t mode) +{ + int err; + errno = 0; + err = mkdir(path, mode); + if (err != 0 && errno != EEXIST) + reel_fail("PCU: could not create directory \"%s\"\n", path); +} + +static void append(char* s, size_t size, const char* format, ...) +{ + int len = strlen(s); + va_list ap; + va_start(ap, format); + vsnprintf(s + len, size - len, format, ap); + va_end(ap); +} + + +void PCU_Debug_Open_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Debug_Open called before Comm_Init"); + + const int fanout = 2048; + const int bufsize = 1024; + char* path = noto_malloc(bufsize); + path[0] = '\0'; + if (PCU_Comm_Peers_2(pcu) > fanout) { + mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; + strcpy(path, "debug/"); + safe_mkdir(path, dir_perm); + int self = PCU_Comm_Self_2(pcu); + append(path, bufsize, "%d/", self / fanout); + if (self % fanout == 0) + safe_mkdir(path, dir_perm); + PCU_Barrier_2(pcu); + } + + append(path,bufsize, "%s", "debug"); + pcu_msg* msg = get_msg(pcu); + if ( ! msg->file) + msg->file = pcu_open_parallel(path,"txt"); + noto_free(path); +} + +void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list ap) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Debug_Print called before Comm_Init"); + pcu_msg* msg = get_msg(pcu); + if ( ! msg->file) + return; //Print is a no-op if no file is open + vfprintf(msg->file,format,ap); + fflush(msg->file); +} +/** \brief like fprintf, contents go to debugN.txt */ +void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) +{ + va_list arglist; + va_start(arglist,format); + PCU_Debug_Printv_2(pcu, format, arglist); + va_end(arglist); +} + +/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ +int PCU_Comm_From_2(pcu_t* pcu, int* from_rank) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_From called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + *from_rank = pcu_order_received_from(m->order); + else + *from_rank = pcu_msg_received_from(m); + return PCU_SUCCESS; +} + +/** \brief Returns in * \a size the bytes in the current received buffer + \details This function should be called after a successful PCU_Comm_Receive. + The size returned will be the total received size regardless of how + much unpacking has been done. + */ +int PCU_Comm_Received_2(pcu_t* pcu, size_t* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Received called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + *size = pcu_order_received_size(m->order); + else + *size = pcu_msg_received_size(m); + return PCU_SUCCESS; +} + +/** \brief Extracts a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Receive. + The next \a size bytes of the current received buffer are unpacked, + and an internal pointer to that data is returned. + The returned pointer must not be freed by the user. + */ +void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Extract called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_unpack(m->order,size); + return pcu_msg_unpack(m,size); +} + +/** \brief Reinitializes PCU with a new MPI communicator. + \details All of PCU's logic is based off two duplicates + of this communicator, so you can safely get PCU to act + on sub-groups of processes using this function. + This call should be collective over all processes + in the previous communicator. This is a very heavy weight function + and should be used sparingly. + */ +void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Switch_Comm called before Comm_Init"); + int result; + MPI_Comm_compare(new_comm, pcu->mpi.original_comm, &result); + if (result != MPI_IDENT) { + pcu_mpi_finalize(&(pcu->mpi)); + pcu_mpi_init(new_comm,&(pcu->mpi)); + } +} + +/** \brief Return the current MPI communicator + \details Returns the communicator given to the + most recent PCU_Switch_Comm call, or MPI_COMM_WORLD + otherwise. + */ +MPI_Comm PCU_Get_Comm_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Get_Comm called before Comm_Init"); + return pcu->mpi.original_comm; +} + +/** \brief Return the time in seconds since some time in the past + */ +double PCU_Time_2(pcu_t* pcu) +{ + return MPI_Wtime(); +} + +void PCU_Protect_2(pcu_t* pcu) +{ + reel_protect(); +} From de2f57d601a4817822acf8a3d1cee97f826f2b83 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 14:21:54 -0400 Subject: [PATCH 007/141] add PCU2.h to install files --- pcu/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 9635d0d40..6041fe095 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -28,6 +28,7 @@ set(SOURCES # Package headers set(HEADERS PCU.h + PCU2.h pcu_io.h pcu_util.h reel/reel.h From 6a461018fe16e1ab39afc7020936a6ad308b3610 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 15:03:06 -0400 Subject: [PATCH 008/141] fix compiler warnings --- pcu/pcu.c | 4 ++-- pcu/pcu2.c | 4 ++++ pcu/pcu_coll.c | 20 ++++++++++++++++++++ pcu/pcu_pmpi.c | 8 ++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index d4679fc86..1e82ff272 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -53,7 +53,7 @@ static pcu_t global_pcu = {.state=pcu_state_uninit}; */ int PCU_Comm_Init(void) { - PCU_Comm_Init_2(&global_pcu); + return PCU_Comm_Init_2(&global_pcu); } /** \brief Frees all PCU library structures. @@ -504,7 +504,7 @@ void PCU_Switch_Comm(MPI_Comm new_comm) */ MPI_Comm PCU_Get_Comm(void) { - PCU_Get_Comm_2(&global_pcu); + return PCU_Get_Comm_2(&global_pcu); } /** \brief Return the time in seconds since some time in the past diff --git a/pcu/pcu2.c b/pcu/pcu2.c index 001ac9ed2..03b4db62d 100644 --- a/pcu/pcu2.c +++ b/pcu/pcu2.c @@ -774,10 +774,14 @@ MPI_Comm PCU_Get_Comm_2(pcu_t* pcu) */ double PCU_Time_2(pcu_t* pcu) { + // silence warning + (void)pcu; return MPI_Wtime(); } void PCU_Protect_2(pcu_t* pcu) { + // silence warning + (void)pcu; reel_protect(); } diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 225113b13..a3ad4e497 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -170,6 +170,8 @@ static bool end_coll_step(pcu_mpi_t* mpi, pcu_coll* c) void pcu_make_coll(pcu_mpi_t* mpi, pcu_coll* c, pcu_pattern* p, pcu_merge* m) { + // silence warning + (void)mpi; c->pattern = p; c->merge = m; } @@ -225,6 +227,8 @@ bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) static int reduce_begin_bit(pcu_mpi_t* mpi) { + // silence warning + (void)mpi; return 1; } @@ -252,6 +256,8 @@ static int reduce_action(pcu_mpi_t* mpi, int bit) static int reduce_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit << 1; } @@ -280,6 +286,8 @@ static int bcast_begin_bit(pcu_mpi_t* mpi) static bool bcast_end_bit(pcu_mpi_t * mpi, int bit) { + // silence warning + (void)mpi; return bit == 0; } @@ -299,6 +307,8 @@ static int bcast_action(pcu_mpi_t* mpi, int bit) static int bcast_shift(pcu_mpi_t * mpi, int bit) { + // silence warning + (void)mpi; return bit >> 1; } @@ -320,6 +330,8 @@ static pcu_pattern bcast = static int scan_up_begin_bit(pcu_mpi_t* mpi) { + // silence warning + (void)mpi; return 1; } @@ -373,6 +385,8 @@ static int scan_up_peer(pcu_mpi_t* mpi, int bit) static int scan_up_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit << 1; } @@ -392,6 +406,8 @@ static int scan_down_begin_bit(pcu_mpi_t* mpi) static bool scan_down_end_bit(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit == 1; } @@ -442,6 +458,8 @@ static int scan_down_peer(pcu_mpi_t * mpi, int bit) static int scan_down_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit >> 1; } @@ -507,6 +525,8 @@ bool pcu_barrier_done(pcu_mpi_t* mpi, pcu_coll* c) void pcu_barrier(pcu_mpi_t* mpi, pcu_coll* c) { + // silence warning + (void)mpi; pcu_begin_barrier(mpi, c); while( ! pcu_barrier_done(mpi, c)); } diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index a4280dace..5bfd3f581 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -48,6 +48,8 @@ void pcu_pmpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { + // silence warning + (void)self; if( m->buffer.size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR PCU message size exceeds INT_MAX... exiting\n"); abort(); @@ -64,6 +66,8 @@ void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) { + // silence warning + (void)self; int flag; MPI_Test(&(m->request),&flag,MPI_STATUS_IGNORE); return flag; @@ -71,11 +75,15 @@ bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) bool pcu_pmpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { + // silence warning + (void)self; return pcu_pmpi_receive2(self, m,0,comm); } bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { + // silence warning + (void)self; MPI_Status status; int flag; MPI_Iprobe(m->peer,tag,comm,&flag,&status); From ed28c0544848070072469cddca0e1862974cbbfb Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 22:11:41 -0400 Subject: [PATCH 009/141] constify mpi functions --- pcu/pcu_mpi.c | 17 ++++++----------- pcu/pcu_mpi.h | 10 +++++----- pcu/pcu_pmpi.c | 18 +++++++++--------- pcu/pcu_pmpi.h | 10 +++++----- 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index 40236ef95..5681d0e90 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -21,41 +21,36 @@ void pcu_free_message(pcu_message* m) pcu_free_buffer(&(m->buffer)); } -int pcu_mpi_size(pcu_mpi_t* self) +int pcu_mpi_size(const pcu_mpi_t* self) { return pcu_pmpi_size(self); } -int pcu_mpi_rank(pcu_mpi_t* self) +int pcu_mpi_rank(const pcu_mpi_t* self) { return pcu_pmpi_rank(self); } -static void check_rank(pcu_mpi_t* self, int rank) +static void check_rank(const pcu_mpi_t* self, int rank) { (void)rank; PCU_ALWAYS_ASSERT(0 <= rank); PCU_ALWAYS_ASSERT(rank < pcu_mpi_size(self)); } -void pcu_mpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +void pcu_mpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { check_rank(self, m->peer); - // verify that the communicator is one of the user or collective communicators - //int user_result, coll_result; - //MPI_Comm_compare(comm, self->user_comm, &user_result); - //MPI_Comm_compare(comm, self->coll_comm, &coll_result); - //PCU_ALWAYS_ASSERT(user_result == MPI_IDENT || coll_result == MPI_IDENT); PCU_ALWAYS_ASSERT(comm == self->user_comm || comm == self->coll_comm); pcu_pmpi_send(self, m, comm); } -bool pcu_mpi_done(pcu_mpi_t* self, pcu_message* m) +bool pcu_mpi_done(const pcu_mpi_t* self, pcu_message* m) { return pcu_pmpi_done(self, m); } -bool pcu_mpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +bool pcu_mpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { if (m->peer != MPI_ANY_SOURCE) check_rank(self, m->peer); diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 9c0fad4f4..fb55ea275 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -32,11 +32,11 @@ typedef struct int size; } pcu_mpi_t; -int pcu_mpi_size(pcu_mpi_t*); -int pcu_mpi_rank(pcu_mpi_t*); -void pcu_mpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_mpi_done(pcu_mpi_t*, pcu_message* m); -bool pcu_mpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +int pcu_mpi_size(const pcu_mpi_t*); +int pcu_mpi_rank(const pcu_mpi_t*); +void pcu_mpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_mpi_done(const pcu_mpi_t*, pcu_message* m); +bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); void pcu_mpi_finalize(pcu_mpi_t* mpi); diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 5bfd3f581..1db376d2c 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -13,8 +13,8 @@ #include #include -void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); +void pcu_pmpi_send2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); +bool pcu_pmpi_receive2(const pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* self) { @@ -31,22 +31,22 @@ void pcu_pmpi_finalize(pcu_mpi_t* self) MPI_Comm_free(&(self->coll_comm)); } -int pcu_pmpi_size(pcu_mpi_t* self) +int pcu_pmpi_size(const pcu_mpi_t* self) { return self->size; } -int pcu_pmpi_rank(pcu_mpi_t* self) +int pcu_pmpi_rank(const pcu_mpi_t* self) { return self->rank; } -void pcu_pmpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +void pcu_pmpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { pcu_pmpi_send2(self, m,0,comm); } -void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) +void pcu_pmpi_send2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { // silence warning (void)self; @@ -64,7 +64,7 @@ void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) &(m->request)); } -bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) +bool pcu_pmpi_done(const pcu_mpi_t* self, pcu_message* m) { // silence warning (void)self; @@ -73,14 +73,14 @@ bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) return flag; } -bool pcu_pmpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +bool pcu_pmpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { // silence warning (void)self; return pcu_pmpi_receive2(self, m,0,comm); } -bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) +bool pcu_pmpi_receive2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { // silence warning (void)self; diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index 36507421d..d6dd08dae 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -16,10 +16,10 @@ void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* mpi); void pcu_pmpi_finalize(pcu_mpi_t* m); -int pcu_pmpi_size(pcu_mpi_t* self); -int pcu_pmpi_rank(pcu_mpi_t* self); -void pcu_pmpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_done(pcu_mpi_t*, pcu_message* m); +int pcu_pmpi_size(const pcu_mpi_t* self); +int pcu_pmpi_rank(const pcu_mpi_t* self); +void pcu_pmpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_done(const pcu_mpi_t*, pcu_message* m); #endif From 2eada4eadad36a7e7b70173b003077448800b20f Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 02:11:09 -0400 Subject: [PATCH 010/141] first pass of PCU with making use of a high level object --- pcu/CMakeLists.txt | 9 +- pcu/PCU.h | 12 +- pcu/PCU2.h | 137 ------- pcu/PCUObj.cc | 276 +++++++++++++++ pcu/PCUObj.h | 101 ++++++ pcu/noto/noto_malloc.h | 23 +- pcu/{pcu.c => pcu.cc} | 466 +++++++++++++----------- pcu/pcu2.c | 787 ----------------------------------------- pcu/pcu_buffer.h | 6 + pcu/pcu_byteorder.h | 8 + pcu/pcu_coll.c | 1 + pcu/pcu_coll.h | 7 + pcu/pcu_defines.h | 13 + pcu/pcu_mem.c | 13 +- pcu/pcu_mem.h | 13 + pcu/pcu_mpi.h | 12 +- pcu/pcu_msg.h | 7 + pcu/pcu_order.h | 6 + pcu/pcu_pmpi.h | 27 +- pcu/pcu_util.h | 36 +- pcu/pkg_tribits.cmake | 39 +- 21 files changed, 798 insertions(+), 1201 deletions(-) delete mode 100644 pcu/PCU2.h create mode 100644 pcu/PCUObj.cc create mode 100644 pcu/PCUObj.h rename pcu/{pcu.c => pcu.cc} (51%) delete mode 100644 pcu/pcu2.c create mode 100644 pcu/pcu_defines.h create mode 100644 pcu/pcu_mem.h diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 6041fe095..fd5055415 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,8 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.c - pcu2.c + pcu.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -23,15 +22,15 @@ set(SOURCES pcu_util.c noto/noto_malloc.c reel/reel.c -) + PCUObj.cc) # Package headers set(HEADERS PCU.h - PCU2.h - pcu_io.h + pcu_io.h pcu_util.h reel/reel.h + pcu_defines.h ) # Add the pcu library diff --git a/pcu/PCU.h b/pcu/PCU.h index f9a53362e..02ff1255a 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -9,10 +9,7 @@ *******************************************************************************/ #ifndef PCU_H #define PCU_H - -#define PCU_SUCCESS 0 -#define PCU_FAILURE -1 - +#include "pcu_defines.h" #include #ifdef __cplusplus @@ -94,13 +91,8 @@ bool PCU_Comm_Read(int* from_rank, void** data, size_t* size); /*Debug file I/O API*/ void PCU_Debug_Open(void); -#ifdef __GNUC__ -void PCU_Debug_Print(const char* format, ...) - __attribute__((format(printf,1,2))); -#else -void PCU_Debug_Print(const char* format, ...); -#endif +void PCU_Debug_Print(const char* format, ...) PCU_FORMAT_ATTRIBUTE(1,2); /*lesser-used APIs*/ bool PCU_Comm_Initialized(void); int PCU_Comm_Packed(int to_rank, size_t* size); diff --git a/pcu/PCU2.h b/pcu/PCU2.h deleted file mode 100644 index 3601ff029..000000000 --- a/pcu/PCU2.h +++ /dev/null @@ -1,137 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -#ifndef PCU2_H -#define PCU2_H - -#define PCU_SUCCESS 0 -#define PCU_FAILURE -1 - -#include - -#ifdef __cplusplus -#include -#include -extern "C" { -#else -#include -#include -#include -#endif - -// FOR pcu_state_type ... use opaque ptr instead? -#include "pcu_msg.h" - -typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; -typedef struct { - pcu_state_enum state; - pcu_msg pmsg; - pcu_mpi_t mpi; -} pcu_t; - -/*library init/finalize*/ -int PCU_Comm_Init_2(pcu_t* pcu); -int PCU_Comm_Free_2(pcu_t* pcu); - -/*rank/size functions*/ -int PCU_Comm_Self_2(pcu_t* pcu); -int PCU_Comm_Peers_2(pcu_t* pcu); - -/*recommended message passing API*/ -void PCU_Comm_Begin_2(pcu_t* pcu); -int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size); -int PCU_Comm_Send_2(pcu_t* pcu); -bool PCU_Comm_Receive_2(pcu_t* pcu); -bool PCU_Comm_Listen_2(pcu_t* pcu); -int PCU_Comm_Sender_2(pcu_t* pcu); -bool PCU_Comm_Unpacked_2(pcu_t* pcu); -int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size); - -/*turns deterministic ordering for the - above API on/off*/ -void PCU_Comm_Order_2(pcu_t* pcu, bool on); - -/*collective operations*/ -void PCU_Barrier_2(pcu_t* pcu); -void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n); -double PCU_Add_Double_2(pcu_t* pcu, double x); -void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n); -double PCU_Min_Double_2(pcu_t* pcu, double x); -void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n); -double PCU_Max_Double_2(pcu_t* pcu, double x); -void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Add_Int_2(pcu_t* pcu, int x); -void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n); -long PCU_Add_Long_2(pcu_t* pcu, long x); -void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Exscan_Int_2(pcu_t* pcu, int x); -void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n); -long PCU_Exscan_Long_2(pcu_t* pcu, long x); -void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); -size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x); -void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); -size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x); -void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); -size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x); -void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Min_Int_2(pcu_t* pcu, int x); -void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Max_Int_2(pcu_t* pcu, int x); -void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n); -long PCU_Max_Long_2(pcu_t* pcu, long x); -int PCU_Or_2(pcu_t* pcu, int c); -int PCU_And_2(pcu_t* pcu, int c); - -/*process-level self/peers (mpi wrappers)*/ -int PCU_Proc_Self_2(pcu_t* pcu); -int PCU_Proc_Peers_2(pcu_t* pcu); - -/*IPComMan replacement API*/ -int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size); -bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size); - -/*Debug file I/O API*/ -void PCU_Debug_Open_2(pcu_t* pcu); -void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list args); -#ifdef __GNUC__ -void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) - __attribute__((format(printf,2,3))); -#else -void PCU_Debug_Print(pcu_t* pcu, const char* format, ...); -#endif - -/*lesser-used APIs*/ -bool PCU_Comm_Initialized_2(pcu_t* pcu); -int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size); -int PCU_Comm_From_2(pcu_t* pcu, int* from_rank); -int PCU_Comm_Received_2(pcu_t* pcu, size_t* size); -void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size); -int PCU_Comm_Rank_2(pcu_t* pcu, int* rank); -int PCU_Comm_Size_2(pcu_t* pcu, int* size); - -int PCU_Comm_Start_2(pcu_t* pcu); - -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm_2(pcu_t* pcu); - -/*stack trace helpers using GNU/Linux*/ -void PCU_Protect_2(pcu_t* pcu); - -/*MPI_Wtime_() equivalent*/ -double PCU_Time_2(pcu_t* pcu); - -/*Memory usage*/ -double PCU_GetMem_2(pcu_t* pcu); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc new file mode 100644 index 000000000..a6a3f9b76 --- /dev/null +++ b/pcu/PCUObj.cc @@ -0,0 +1,276 @@ +#include "PCUObj.h" +#include "noto_malloc.h" +#include "pcu_mem.h" +#include "pcu_mpi.h" +#include "pcu_msg.h" +#include "pcu_order.h" +#include "reel.h" +#include +#include /*using POSIX mkdir call for SMB "foo/" path*/ +namespace pcu { + +int PCU::Peers() const noexcept { return pcu_mpi_size(mpi_); } +int PCU::Self() const noexcept { return pcu_mpi_rank(mpi_); } +void PCU::Begin() noexcept { pcu_msg_start(mpi_, msg_); } +int PCU::Pack(int to_rank, const void *data, size_t size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Pack"); + if (size > (size_t)INT_MAX) { + fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds " + "INT_MAX... exiting\n"); + abort(); + } + memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); + return PCU_SUCCESS; +} +int PCU::Send() noexcept { + pcu_msg_send(mpi_, msg_); + return PCU_SUCCESS; +} +bool PCU::Receive(void) noexcept { + while (Unpacked()) + if (!Listen()) + return false; + return true; +} +bool PCU::Listen() noexcept { + if (msg_->order) + return pcu_order_receive(mpi_, msg_->order, msg_); + return pcu_msg_receive(mpi_, msg_); +} +int PCU::Sender() noexcept { + if (msg_->order) + return pcu_order_received_from(msg_->order); + return pcu_msg_received_from(msg_); +} +bool PCU::Unpacked() noexcept { + if (msg_->order) + return pcu_order_unpacked(msg_->order); + return pcu_msg_unpacked(msg_); +} +int PCU::Unpack(void *data, size_t size) noexcept { + if (msg_->order) + memcpy(data, pcu_order_unpack(msg_->order, size), size); + else + memcpy(data, pcu_msg_unpack(msg_, size), size); + return PCU_SUCCESS; +} +int PCU::Write(int to_rank, const void *data, size_t size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Write"); + PCU_MSG_PACK(msg_, to_rank, size); + memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); + return PCU_SUCCESS; +} +bool PCU::Read(int *from_rank, void **data, size_t *size) noexcept { + if (!Receive()) + return false; + *from_rank = Sender(); + Unpack(size, sizeof(*size)); + *data = Extract(*size); + return true; +} +void PCU::Order(bool on) { + if (on && (!msg_->order)) + msg_->order = pcu_order_new(); + if ((!on) && msg_->order) { + pcu_order_free(msg_->order); + msg_->order = NULL; + } +} +void PCU::Barrier() { pcu_barrier(mpi_, &(msg_->coll)); } +int PCU::Or(int c) noexcept { return Max(c); } +int PCU::And(int c) noexcept { return Min(c); } +int PCU::Packed(int to_rank, size_t *size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Packed"); + *size = pcu_msg_packed(msg_, to_rank); + return PCU_SUCCESS; +} +int PCU::From(int *from_rank) noexcept { + if (msg_->order) + *from_rank = pcu_order_received_from(msg_->order); + else + *from_rank = pcu_msg_received_from(msg_); + return PCU_SUCCESS; +} +int PCU::Received(size_t *size) noexcept { + if (msg_->order) + *size = pcu_order_received_size(msg_->order); + else + *size = pcu_msg_received_size(msg_); + return PCU_SUCCESS; +} +void *PCU::Extract(size_t size) noexcept { + if (msg_->order) + return pcu_order_unpack(msg_->order, size); + return pcu_msg_unpack(msg_, size); +} + +static void safe_mkdir(const char *path, mode_t mode) { + int err; + errno = 0; + err = mkdir(path, mode); + if (err != 0 && errno != EEXIST) + reel_fail("PCU: could not create directory \"%s\"\n", path); +} + +static void append(char *s, size_t size, const char *format, ...) { + int len = strlen(s); + va_list ap; + va_start(ap, format); + vsnprintf(s + len, size - len, format, ap); + va_end(ap); +} + +void PCU::DebugOpen() noexcept { + const int fanout = 2048; + const int bufsize = 1024; + char *path = (char *)noto_malloc(bufsize); + path[0] = '\0'; + if (Peers() > fanout) { + mode_t const dir_perm = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; + strcpy(path, "debug/"); + safe_mkdir(path, dir_perm); + int self = Self(); + append(path, bufsize, "%d/", self / fanout); + if (self % fanout == 0) + safe_mkdir(path, dir_perm); + Barrier(); + } + + append(path, bufsize, "%s", "debug"); + if (!msg_->file) + msg_->file = pcu_open_parallel(path, "txt"); + noto_free(path); +} + +double GetMem(void) noexcept { return pcu_get_mem(); } +void Protect(void) noexcept { reel_protect(); } +double Time(void) noexcept { return MPI_Wtime(); } + +void PCU::DebugPrint(const char *format, ...) noexcept { + va_list args; + va_start(args, format); + DebugPrint(format, args); + va_end(args); +} +void PCU::DebugPrint(const char *format, va_list args) noexcept { + if (!msg_->file) + return; // Print is a no-op if no file is open + vfprintf(msg_->file, format, args); + fflush(msg_->file); +} +PCU::PCU(MPI_Comm comm) { + mpi_ = new pcu_mpi_t; + msg_ = new pcu_msg; + pcu_mpi_init(comm, mpi_); + pcu_make_msg(msg_); + /* turn ordering on by default, call + PCU_Comm_Order(false) after PCU_Comm_Init + to disable this */ + Order(true); +} +PCU::~PCU() noexcept { + delete mpi_; + delete msg_; +} +PCU::PCU(PCU &&other) noexcept { + std::swap(mpi_, other.mpi_); + std::swap(msg_, other.msg_); +} +PCU &PCU::operator=(PCU && other) noexcept { + std::swap(mpi_, other.mpi_); + std::swap(msg_, other.msg_); + return *this; +} +MPI_Comm PCU::GetMPIComm() const noexcept { return mpi_->original_comm; } + +/* template implementations */ +template void PCU::Add(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] += b[i]; + }, + p, n * sizeof(T)); +} +template T PCU::Add(T p) noexcept { + Add(&p, 1); + return p; +} +template void PCU::Min(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] = std::min(a[i], b[i]); + }, + p, n * sizeof(T)); +} +template T PCU::Min(T p) noexcept { + Min(&p, 1); + return p; +} +template void PCU::Max(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] = std::max(a[i], b[i]); + }, + p, n * sizeof(T)); +} +template T PCU::Max(T p) noexcept { + Max(&p, 1); + return p; +} +template void PCU::Exscan(T *p, size_t n) noexcept { + auto *originals = (T *)noto_malloc(sizeof(T) * n); + for (size_t i = 0; i < n; ++i) + originals[i] = p[i]; + pcu_scan( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] += b[i]; + }, + p, n * sizeof(T)); + // convert inclusive scan to exclusive + for (size_t i = 0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} +template T PCU::Exscan(T p) noexcept { + Exscan(&p, 1); + return p; +} +#define PCU_EXPL_INST_DECL(T) \ + template void PCU::Add(T * p, size_t n) noexcept; \ + template T PCU::Add(T p) noexcept; \ + template void PCU::Min(T * p, size_t n) noexcept; \ + template T PCU::Min(T p) noexcept; \ + template void PCU::Max(T * p, size_t n) noexcept; \ + template T PCU::Max(T p) noexcept; \ + template void PCU::Exscan(T * p, size_t n) noexcept; \ + template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL + +} // namespace pcu diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h new file mode 100644 index 000000000..9584f5b6c --- /dev/null +++ b/pcu/PCUObj.h @@ -0,0 +1,101 @@ +#ifndef SCOREC_PCU_PCUOBJ_H +#define SCOREC_PCU_PCUOBJ_H +#include +#include +#include "pcu_defines.h" + +struct pcu_msg_struct; +struct pcu_mpi_struct; + +namespace pcu { +class PCU { +public: + explicit PCU(MPI_Comm comm); + ~PCU() noexcept; + PCU(PCU const &) = delete; + PCU(PCU &&) noexcept; + PCU &operator=(PCU const &) = delete; + PCU &operator=(PCU &&) noexcept; + /** @brief Returns the rank of the current process. + * @return The rank of the current process. + */ + [[nodiscard]] int Self() const noexcept; + /** @brief Returns the number of ranks in the communicator. + * @return The number of ranks in the communicator. + */ + [[nodiscard]] int Peers() const noexcept; + [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; + + /*recommended message passing API*/ + void Begin() noexcept; + int Pack(int to_rank, const void *data, size_t size) noexcept; + int Send() noexcept; + bool Receive() noexcept; + bool Listen() noexcept; + int Sender() noexcept; + bool Unpacked() noexcept; + int Unpack(void *data, size_t size) noexcept; + /*IPComMan replacement API*/ + int Write(int to_rank, const void *data, size_t size) noexcept; + bool Read(int *from_rank, void **data, size_t *size) noexcept; + + /*turns deterministic ordering for the + above API on/off*/ + void Order(bool on); + + /*collective operations*/ + void Barrier(); + template void Add(T *p, size_t n) noexcept; + template [[nodiscard]] T Add(T p) noexcept; + template void Min(T *p, size_t n) noexcept; + template [[nodiscard]] T Min(T p) noexcept; + template void Max(T *p, size_t n) noexcept; + template [[nodiscard]] T Max(T p) noexcept; + template void Exscan(T *p, size_t n) noexcept; + template [[nodiscard]] T Exscan(T p) noexcept; + + /*bitwise operations*/ + [[nodiscard]] int Or(int c) noexcept; + [[nodiscard]] int And(int c) noexcept; + + /*lesser-used APIs*/ + int Packed(int to_rank, size_t *size) noexcept; + int From(int *from_rank) noexcept; + int Received(size_t *size) noexcept; + void *Extract(size_t size) noexcept; + + void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3); + void DebugPrint(const char* format, va_list args) noexcept; + /* Debug functions */ + void DebugOpen() noexcept; + +private: + pcu_msg_struct *msg_; + pcu_mpi_struct *mpi_; +}; +/*stack trace helpers using GNU/Linux*/ +void Protect(void) noexcept; +/*Memory usage*/ +[[nodiscard]] double GetMem(void) noexcept; +/*MPI_Wtime() equivalent*/ +[[nodiscard]] double Time(void) noexcept; + +/* explicit instantiations of template functions */ +#define PCU_EXPL_INST_DECL(T) \ + extern template void PCU::Add(T * p, size_t n) noexcept; \ + extern template T PCU::Add(T p) noexcept; \ + extern template void PCU::Min(T * p, size_t n) noexcept; \ + extern template T PCU::Min(T p) noexcept; \ + extern template void PCU::Max(T * p, size_t n) noexcept; \ + extern template T PCU::Max(T p) noexcept; \ + extern template void PCU::Exscan(T * p, size_t n) noexcept; \ + extern template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL + +} // namespace pcu +#undef PCU_FORMAT_ATTRIBUTE +#endif // SCOREC_PCU_PCUOBJ_H diff --git a/pcu/noto/noto_malloc.h b/pcu/noto/noto_malloc.h index e1f60ed36..74a2c4bac 100644 --- a/pcu/noto/noto_malloc.h +++ b/pcu/noto/noto_malloc.h @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2015 Scientific Computation Research Center, + Copyright 2015 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -10,13 +10,18 @@ #ifndef NOTO_MALLOC_H #define NOTO_MALLOC_H +#ifdef __cplusplus +extern "C" { +#endif + #include -void* noto_malloc(size_t size); -#define NOTO_MALLOC(p,c) ((p)=noto_malloc(sizeof(*(p))*(c))) -void* noto_realloc(void* p, size_t size); -void noto_free(void* p); +void *noto_malloc(size_t size); +#define NOTO_MALLOC(p, c) ((p) = noto_malloc(sizeof(*(p)) * (c))) +void *noto_realloc(void *p, size_t size); +void noto_free(void *p); size_t noto_malloced(void); - +#ifdef __cplusplus +} +#endif #endif - diff --git a/pcu/pcu.c b/pcu/pcu.cc similarity index 51% rename from pcu/pcu.c rename to pcu/pcu.cc index 1e82ff272..3a0337812 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.cc @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2011 Scientific Computation Research Center, + Copyright 2011 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -29,40 +29,35 @@ The API documentation is here: pcu.c */ -#include -#include #include "PCU.h" -#include "PCU2.h" -#include "pcu_msg.h" -#include "pcu_order.h" -#include "noto_malloc.h" +#include "PCUObj.h" #include "reel.h" -#include /*required for mode_t for mkdir on some systems*/ -#include /*using POSIX mkdir call for SMB "foo/" path*/ -#include /* for checking the error from mkdir */ -#include /*INT_MAX*/ -#include /*abort*/ - - -static pcu_t global_pcu = {.state=pcu_state_uninit}; +#include +static pcu::PCU *global_pcu = nullptr; +extern "C" { /** \brief Initializes the PCU library. \details This function must be called by all MPI processes before calling any other PCU functions. MPI_Init or MPI_Init_thread should be called before this function. */ -int PCU_Comm_Init(void) -{ - return PCU_Comm_Init_2(&global_pcu); +int PCU_Comm_Init(void) { + if (global_pcu != nullptr) + reel_fail("nested calls to Comm_Init"); + global_pcu = new pcu::PCU(MPI_COMM_WORLD); + return PCU_SUCCESS; } /** \brief Frees all PCU library structures. \details This function must be called by all MPI processes after all other calls to PCU, and before calling MPI_Finalize. */ -int PCU_Comm_Free(void) -{ - return PCU_Comm_Free_2(&global_pcu); +int PCU_Comm_Free(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Free called before Comm_Init"); + delete global_pcu; + global_pcu = nullptr; + return PCU_SUCCESS; } /** \brief Returns the communication rank of the calling thread. @@ -74,18 +69,20 @@ int PCU_Comm_Free(void) Ranks are contiguous within a process, so that the \f$t\f$ threads in process \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. */ -int PCU_Comm_Self(void) -{ - return PCU_Comm_Self_2(&global_pcu); +int PCU_Comm_Self(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Self called before Comm_Init"); + return global_pcu->Self(); } /** \brief Returns the number of threads in the program. \details when called from a non-threaded MPI process, this function is equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). */ -int PCU_Comm_Peers(void) -{ - return PCU_Comm_Peers_2(&global_pcu); +int PCU_Comm_Peers(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Peers called before Comm_Init"); + return global_pcu->Peers(); } /** \brief Begins a PCU communication phase. @@ -94,9 +91,10 @@ int PCU_Comm_Peers(void) After calling this function, each thread may call functions like PCU_Comm_Pack or PCU_Comm_Write. */ -void PCU_Comm_Begin(void) -{ - PCU_Comm_Begin_2(&global_pcu); +void PCU_Comm_Begin(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Begin called before Comm_Init"); + global_pcu->Begin(); } /** \brief Packs data to be sent to \a to_rank. @@ -105,9 +103,10 @@ void PCU_Comm_Begin(void) This function should be called after PCU_Comm_Start and before PCU_Comm_Send. */ -int PCU_Comm_Pack(int to_rank, const void* data, size_t size) -{ - return PCU_Comm_Pack_2(&global_pcu, to_rank, data, size); +int PCU_Comm_Pack(int to_rank, const void *data, size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Pack called before Comm_Init"); + return global_pcu->Pack(to_rank, data, size); } /** \brief Sends all buffers for this communication phase. @@ -117,9 +116,10 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) All buffers from this thread are sent out and receiving may begin after this call. */ -int PCU_Comm_Send(void) -{ - return PCU_Comm_Send_2(&global_pcu); +int PCU_Comm_Send(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Send called before Comm_Init"); + return global_pcu->Send(); } /** \brief Tries to receive a buffer for this communication phase. @@ -133,25 +133,28 @@ int PCU_Comm_Send(void) Users should unpack all data from this buffer before calling this function again, because the previously received buffer is destroyed by the call. */ -bool PCU_Comm_Listen(void) -{ - return PCU_Comm_Listen_2(&global_pcu); +bool PCU_Comm_Listen(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Listen called before Comm_Init"); + return global_pcu->Listen(); } /** \brief Returns in * \a from_rank the sender of the current received buffer. \details This function should be called after a successful PCU_Comm_Listen. */ -int PCU_Comm_Sender(void) -{ - return PCU_Comm_Sender_2(&global_pcu); +int PCU_Comm_Sender(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Sender called before Comm_Init"); + return global_pcu->Sender(); } /** \brief Returns true if the current received buffer has been unpacked. \details This function should be called after a successful PCU_Comm_Listen. */ -bool PCU_Comm_Unpacked(void) -{ - return PCU_Comm_Unpacked_2(&global_pcu); +bool PCU_Comm_Unpacked(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Unpacked called before Comm_Init"); + return global_pcu->Unpacked(); } /** \brief Unpacks a block of data from the current received buffer. @@ -164,20 +167,23 @@ bool PCU_Comm_Unpacked(void) Users must ensure that there remain \a size bytes to be unpacked, PCU_Comm_Unpacked can help with this. */ -int PCU_Comm_Unpack(void* data, size_t size) -{ - return PCU_Comm_Unpack_2(&global_pcu, data, size); +int PCU_Comm_Unpack(void *data, size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Unpack called before Comm_Init"); + return global_pcu->Unpack(data, size); } -void PCU_Comm_Order(bool on) -{ - PCU_Comm_Order_2(&global_pcu, on); +void PCU_Comm_Order(bool on) { + if (global_pcu == nullptr) + reel_fail("Comm_Order called before Comm_Init"); + global_pcu->Order(on); } /** \brief Blocking barrier over all threads. */ -void PCU_Barrier(void) -{ - PCU_Barrier_2(&global_pcu); +void PCU_Barrier(void) { + if (global_pcu == nullptr) + reel_fail("Barrier called before Comm_Init"); + global_pcu->Barrier(); } /** \brief Performs an Allreduce sum of double arrays. @@ -186,94 +192,114 @@ void PCU_Barrier(void) After this call, p[i] will contain the sum of all p[i]'s given by each rank. */ -void PCU_Add_Doubles(double* p, size_t n) -{ - PCU_Add_Doubles_2(&global_pcu, p, n); +void PCU_Add_Doubles(double *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_Doubles called before Comm_Init"); + global_pcu->Add(p, n); } -double PCU_Add_Double(double x) -{ - return PCU_Add_Double_2(&global_pcu, x); +double PCU_Add_Double(double x) { + if (global_pcu == nullptr) + reel_fail("Add_Double called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce minimum of double arrays. - */ -void PCU_Min_Doubles(double* p, size_t n) -{ - PCU_Min_Doubles_2(&global_pcu, p, n); + */ +void PCU_Min_Doubles(double *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Min_Doubles called before Comm_Init"); + global_pcu->Min(p, n); } -double PCU_Min_Double(double x) -{ - return PCU_Min_Double_2(&global_pcu, x); +double PCU_Min_Double(double x) { + if (global_pcu == nullptr) + reel_fail("Min_Double called before Comm_Init"); + return global_pcu->Min(x); } /** \brief Performs an Allreduce maximum of double arrays. - */ -void PCU_Max_Doubles(double* p, size_t n) -{ - PCU_Max_Doubles_2(&global_pcu, p, n); + */ +void PCU_Max_Doubles(double *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_Doubles called before Comm_Init"); + global_pcu->Max(p, n); } -double PCU_Max_Double(double x) -{ - return PCU_Max_Double_2(&global_pcu, x); +double PCU_Max_Double(double x) { + if (global_pcu == nullptr) + reel_fail("Max_Double called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs an Allreduce sum of integers - */ -void PCU_Add_Ints(int* p, size_t n) -{ - PCU_Add_Ints_2(&global_pcu, p, n); + */ +void PCU_Add_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_Ints called before Comm_Init"); + global_pcu->Add(p, n); } -int PCU_Add_Int(int x) -{ - return PCU_Add_Int_2(&global_pcu, x); +int PCU_Add_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Add_Int called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce sum of long integers - */ -void PCU_Add_Longs(long* p, size_t n) -{ - PCU_Add_Longs_2(&global_pcu, p, n); + */ +void PCU_Add_Longs(long *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_Longs called before Comm_Init"); + global_pcu->Add(p, n); } -long PCU_Add_Long(long x) -{ - return PCU_Add_Long_2(&global_pcu, x); +long PCU_Add_Long(long x) { + if (global_pcu == nullptr) + reel_fail("Add_Long called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce sum of size_t unsigned integers - */ -void PCU_Add_SizeTs(size_t* p, size_t n) -{ - PCU_Add_SizeTs_2(&global_pcu, p, n); + */ +void PCU_Add_SizeTs(size_t *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_SizeTs called before Comm_Init"); + global_pcu->Add(p, n); } -size_t PCU_Add_SizeT(size_t x) -{ - return PCU_Add_SizeT_2(&global_pcu, x); +size_t PCU_Add_SizeT(size_t x) { + if (global_pcu == nullptr) + reel_fail("Add_SizeT called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce minimum of size_t unsigned integers - */ -void PCU_Min_SizeTs(size_t* p, size_t n) { - PCU_Min_SizeTs_2(&global_pcu, p, n); + */ +void PCU_Min_SizeTs(size_t *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Min_SizeTs called before Comm_Init"); + global_pcu->Min(p, n); } size_t PCU_Min_SizeT(size_t x) { - return PCU_Min_SizeT_2(&global_pcu, x); + if (global_pcu == nullptr) + reel_fail("Min_SizeT called before Comm_Init"); + return global_pcu->Min(x); } /** \brief Performs an Allreduce maximum of size_t unsigned integers - */ -void PCU_Max_SizeTs(size_t* p, size_t n) { - PCU_Max_SizeTs_2(&global_pcu, p, n); + */ +void PCU_Max_SizeTs(size_t *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_SizeTs called before Comm_Init"); + global_pcu->Max(p, n); } size_t PCU_Max_SizeT(size_t x) { - return PCU_Max_SizeT_2(&global_pcu, x); + if (global_pcu == nullptr) + reel_fail("Max_SizeT called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs an exclusive prefix sum of integer arrays. @@ -282,115 +308,129 @@ size_t PCU_Max_SizeT(size_t x) { After this call, p[i] will contain the sum of all p[i]'s given by ranks lower than the calling rank. */ -void PCU_Exscan_Ints(int* p, size_t n) -{ - PCU_Exscan_Ints_2(&global_pcu, p, n); +void PCU_Exscan_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Exscan_Ints called before Comm_Init"); + global_pcu->Exscan(p, n); } -int PCU_Exscan_Int(int x) -{ - return PCU_Exscan_Int_2(&global_pcu, x); +int PCU_Exscan_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Exscan_Int called before Comm_Init"); + return global_pcu->Exscan(x); } /** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs(long* p, size_t n) -{ - PCU_Exscan_Longs_2(&global_pcu, p, n); +void PCU_Exscan_Longs(long *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Exscan_Longs called before Comm_Init"); + global_pcu->Exscan(p, n); } -long PCU_Exscan_Long(long x) -{ - return PCU_Exscan_Long_2(&global_pcu, x); +long PCU_Exscan_Long(long x) { + if (global_pcu == nullptr) + reel_fail("Exscan_Long called before Comm_Init"); + return global_pcu->Exscan(x); } /** \brief Performs an Allreduce minimum of int arrays. - */ -void PCU_Min_Ints(int* p, size_t n) -{ - PCU_Min_Ints_2(&global_pcu, p, n); + */ +void PCU_Min_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Min_Ints called before Comm_Init"); + global_pcu->Min(p, n); } -int PCU_Min_Int(int x) -{ - return PCU_Min_Int_2(&global_pcu, x); +int PCU_Min_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Min_Int called before Comm_Init"); + return global_pcu->Min(x); } /** \brief Performs an Allreduce maximum of int arrays. - */ -void PCU_Max_Ints(int* p, size_t n) -{ - PCU_Max_Ints_2(&global_pcu, p, n); + */ +void PCU_Max_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_Ints called before Comm_Init"); + global_pcu->Max(p, n); } -int PCU_Max_Int(int x) -{ - return PCU_Max_Int_2(&global_pcu, x); +int PCU_Max_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Max_Int called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs an Allreduce maximum of long arrays. - */ -void PCU_Max_Longs(long* p, size_t n) -{ - PCU_Max_Longs_2(&global_pcu, p, n); + */ +void PCU_Max_Longs(long *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_Longs called before Comm_Init"); + global_pcu->Max(p, n); } -long PCU_Max_Long(long x) -{ - return PCU_Max_Long_2(&global_pcu, x); +long PCU_Max_Long(long x) { + if (global_pcu == nullptr) + reel_fail("Max_Long called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs a parallel logical OR reduction - */ -int PCU_Or(int c) -{ - return PCU_Or_2(&global_pcu, c); + */ +int PCU_Or(int c) { + if (global_pcu == nullptr) + reel_fail("Or called before Comm_Init"); + return global_pcu->Or(c); } /** \brief Performs a parallel logical AND reduction - */ -int PCU_And(int c) -{ - return PCU_And_2(&global_pcu, c); + */ +int PCU_And(int c) { + if (global_pcu == nullptr) + reel_fail("And called before Comm_Init"); + return global_pcu->And(c); } /** \brief Returns the unique rank of the calling process. */ -int PCU_Proc_Self(void) -{ - return PCU_Proc_Self_2(&global_pcu); +int PCU_Proc_Self(void) { + if (global_pcu == nullptr) + reel_fail("Proc_Self called before Comm_Init"); + return global_pcu->Self(); } /** \brief Returns the number of processes. */ -int PCU_Proc_Peers(void) -{ - return PCU_Proc_Peers_2(&global_pcu); +int PCU_Proc_Peers(void) { + if (global_pcu == nullptr) + reel_fail("Proc_Peers called before Comm_Init"); + return global_pcu->Peers(); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ -int PCU_Comm_Rank(int* rank) -{ - return PCU_Comm_Rank_2(&global_pcu, rank); +int PCU_Comm_Rank(int *rank) { + if (global_pcu == nullptr) + reel_fail("Comm_Rank called before Comm_Init"); + return global_pcu->Self(); } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size(int* size) -{ - return PCU_Comm_Size_2(&global_pcu, size); +int PCU_Comm_Size(int *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Size called before Comm_Init"); + *size = global_pcu->Peers(); + return PCU_SUCCESS; } /** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized(void) -{ - return PCU_Comm_Initialized_2(&global_pcu); -} +bool PCU_Comm_Initialized(void) { return global_pcu != nullptr; } /** \brief Deprecated, see PCU_Comm_Begin. */ -int PCU_Comm_Start(PCU_Method method) -{ - (void)method; //warning silencer - return PCU_Comm_Start_2(&global_pcu); +int PCU_Comm_Start(PCU_Method method) { + (void)method; // warning silencer + global_pcu->Begin(); + return PCU_SUCCESS; } /** \brief Returns in * \a size the number of bytes being sent to \a to_rank. @@ -398,9 +438,10 @@ int PCU_Comm_Start(PCU_Method method) This function should be called after PCU_Comm_Start and before PCU_Comm_Send. */ -int PCU_Comm_Packed(int to_rank, size_t* size) -{ - return PCU_Comm_Packed_2(&global_pcu, to_rank, size); +int PCU_Comm_Packed(int to_rank, size_t *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Packed called before Comm_Init"); + return global_pcu->Packed(to_rank, size); } /** \brief Packs a message to be sent to \a to_rank. @@ -412,15 +453,17 @@ int PCU_Comm_Packed(int to_rank, size_t* size) PCU_Comm_Send. If this function is used, PCU_Comm_Pack should not be used. */ -int PCU_Comm_Write(int to_rank, const void* data, size_t size) -{ - return PCU_Comm_Write_2(&global_pcu, to_rank, data, size); +int PCU_Comm_Write(int to_rank, const void *data, size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Write called before Comm_Init"); + return global_pcu->Pack(to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive(void) -{ - return PCU_Comm_Receive_2(&global_pcu); +bool PCU_Comm_Receive(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Receive called before Comm_Init"); + return global_pcu->Receive(); } /** \brief Receives a message for this communication phase. @@ -435,32 +478,33 @@ bool PCU_Comm_Receive(void) Note that the address * \a data points into a PCU buffer, so it is strongly recommended that this data be read and not modified. */ -bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) -{ - return PCU_Comm_Read_2(&global_pcu, from_rank, data, size); +bool PCU_Comm_Read(int *from_rank, void **data, size_t *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Read called before Comm_Init"); + return global_pcu->Read(from_rank, data, size); } - - - -void PCU_Debug_Open(void) -{ - PCU_Debug_Open_2(&global_pcu); +void PCU_Debug_Open(void) { + if (global_pcu == nullptr) + reel_fail("Debug_Open called before Comm_Init"); + global_pcu->DebugOpen(); } /** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print(const char* format, ...) -{ +void PCU_Debug_Print(const char *format, ...) { + if (global_pcu == nullptr) + reel_fail("Debug_Print called before Comm_Init"); va_list arglist; va_start(arglist, format); - PCU_Debug_Printv_2(&global_pcu, format, arglist); + global_pcu->DebugPrint(format, arglist); va_end(arglist); } /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From(int* from_rank) -{ - return PCU_Comm_From_2(&global_pcu, from_rank); +int PCU_Comm_From(int *from_rank) { + if (global_pcu == nullptr) + reel_fail("Comm_From called before Comm_Init"); + return global_pcu->From(from_rank); } /** \brief Returns in * \a size the bytes in the current received buffer @@ -468,9 +512,10 @@ int PCU_Comm_From(int* from_rank) The size returned will be the total received size regardless of how much unpacking has been done. */ -int PCU_Comm_Received(size_t* size) -{ - return PCU_Comm_Received_2(&global_pcu, size); +int PCU_Comm_Received(size_t *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Received called before Comm_Init"); + return global_pcu->Received(size); } /** \brief Extracts a block of data from the current received buffer. @@ -479,9 +524,10 @@ int PCU_Comm_Received(size_t* size) and an internal pointer to that data is returned. The returned pointer must not be freed by the user. */ -void* PCU_Comm_Extract(size_t size) -{ - return PCU_Comm_Extract_2(&global_pcu, size); +void *PCU_Comm_Extract(size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Extract called before Comm_Init"); + return global_pcu->Extract(size); } /** \brief Reinitializes PCU with a new MPI communicator. @@ -492,9 +538,13 @@ void* PCU_Comm_Extract(size_t size) in the previous communicator. This is a very heavy weight function and should be used sparingly. */ -void PCU_Switch_Comm(MPI_Comm new_comm) -{ - PCU_Switch_Comm_2(&global_pcu, new_comm); +void PCU_Switch_Comm(MPI_Comm new_comm) { + if (global_pcu == nullptr) + reel_fail("Switch_Comm called before Comm_Init"); + if (new_comm != global_pcu->GetMPIComm()) { + delete global_pcu; + global_pcu = new pcu::PCU(new_comm); + } } /** \brief Return the current MPI communicator @@ -502,19 +552,17 @@ void PCU_Switch_Comm(MPI_Comm new_comm) most recent PCU_Switch_Comm call, or MPI_COMM_WORLD otherwise. */ -MPI_Comm PCU_Get_Comm(void) -{ - return PCU_Get_Comm_2(&global_pcu); +MPI_Comm PCU_Get_Comm(void) { + if (global_pcu == nullptr) + reel_fail("Get_Comm called before Comm_Init"); + return global_pcu->GetMPIComm(); } /** \brief Return the time in seconds since some time in the past */ -double PCU_Time(void) -{ - return PCU_Time_2(&global_pcu); -} +double PCU_Time(void) { return pcu::Time(); } + +void PCU_Protect(void) { return pcu::Protect(); } -void PCU_Protect(void) -{ - PCU_Protect_2(&global_pcu); +double PCU_GetMem(void) { return pcu::GetMem(); } } diff --git a/pcu/pcu2.c b/pcu/pcu2.c deleted file mode 100644 index 03b4db62d..000000000 --- a/pcu/pcu2.c +++ /dev/null @@ -1,787 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -/** \file pcu->c - \brief The PCU communication interface */ -/** \page pcu PCU - PCU (the Parallel Control Utility) is a library for parallel computation - based on MPI. - PCU provides three things to users: - 1. A hybrid phased message passing system - 2. Hybrid collective operations - - Phased message passing is similar to Bulk Synchronous Parallel. - All messages are exchanged in a phase, which is a collective operation - involving all threads in the parallel program. - During a phase, the following events happen in sequence: - 1. All threads send non-blocking messages to other threads - 2. All threads receive all messages sent to them during this phase - PCU provides termination detection, which is the ability to detect when all - messages have been received without prior knowledge of which threads - are sending to which. - - The API documentation is here: pcu->c -*/ - -#include -#include -#include "PCU2.h" -#include "pcu_msg.h" -#include "pcu_order.h" -#include "noto_malloc.h" -#include "reel.h" -#include /*required for mode_t for mkdir on some systems*/ -#include /*using POSIX mkdir call for SMB "foo/" path*/ -#include /* for checking the error from mkdir */ -#include /*INT_MAX*/ -#include /*abort*/ - - -static pcu_msg* get_msg(pcu_t * pcu) -{ - return &(pcu->pmsg); -} - -/** \brief Initializes the PCU library. - \details This function must be called by all MPI processes before - calling any other PCU functions. - MPI_Init or MPI_Init_thread should be called before this function. - */ -int PCU_Comm_Init_2(pcu_t* pcu) -{ - if (pcu->state != pcu_state_uninit) - reel_fail("nested calls to Comm_Init"); - pcu_mpi_init(MPI_COMM_WORLD,&(pcu->mpi)); - pcu_make_msg(&(pcu->pmsg)); - pcu->state = pcu_state_init; - /* turn ordering on by default, call - PCU_Comm_Order(false) after PCU_Comm_Init - to disable this */ - PCU_Comm_Order_2(pcu, true); - return PCU_SUCCESS; -} - -/** \brief Frees all PCU library structures. - \details This function must be called by all MPI processes after all - other calls to PCU, and before calling MPI_Finalize. - */ -int PCU_Comm_Free_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Free called before Comm_Init"); - if (pcu->pmsg.order) - pcu_order_free(pcu->pmsg.order); - pcu_free_msg(&(pcu->pmsg)); - pcu_mpi_finalize(&(pcu->mpi)); - pcu->state = pcu_state_uninit; - return PCU_SUCCESS; -} - -/** \brief Returns the communication rank of the calling thread. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_rank(MPI_COMM_WORLD,rank). - - Ranks are consecutive from 0 to \f$pt-1\f$ for a program with - \f$p\f$ processes and \f$t\f$ threads per process. - Ranks are contiguous within a process, so that the \f$t\f$ threads in process - \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. - */ -int PCU_Comm_Self_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(&(pcu->mpi)); -} - -/** \brief Returns the number of threads in the program. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). - */ -int PCU_Comm_Peers_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(&(pcu->mpi)); -} - -/** \brief Begins a PCU communication phase. - \details This function must be called by all threads in the MPI job - at the beginning of each phase of communication. - After calling this function, each thread may call functions like - PCU_Comm_Pack or PCU_Comm_Write. -*/ -void PCU_Comm_Begin_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(&(pcu->mpi), get_msg(pcu)); -} - -/** \brief Packs data to be sent to \a to_rank. - \details This function appends the block of \a size bytes starting - at \a data to the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) - reel_fail("Invalid rank in Comm_Pack"); - if ( size > (size_t)INT_MAX ) { - fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); - abort(); - } - memcpy(pcu_msg_pack(get_msg(pcu),to_rank,size),data,size); - return PCU_SUCCESS; -} - -/** \brief Sends all buffers for this communication phase. - \details This function should be called by all threads in the MPI job - after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls - to PCU_Comm_Listen or PCU_Comm_Read. - All buffers from this thread are sent out and receiving - may begin after this call. - */ -int PCU_Comm_Send_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(&(pcu->mpi), get_msg(pcu)); - return PCU_SUCCESS; -} - -/** \brief Tries to receive a buffer for this communication phase. - \details Either this function or PCU_Comm_Read should be called at least - once by all threads during the communication phase, after PCU_Comm_Send - is called. The result will be false if and only if the communication phase - is over and there are no more buffers to receive. - Otherwise, a buffer was received. - Its contents are retrievable through PCU_Comm_Unpack, and its metadata through - PCU_Comm_Sender and PCU_Comm_Received. - Users should unpack all data from this buffer before calling this function - again, because the previously received buffer is destroyed by the call. - */ -bool PCU_Comm_Listen_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Listen called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_receive(&(pcu->mpi), m->order, m); - return pcu_msg_receive(&(pcu->mpi), m); -} - -/** \brief Returns in * \a from_rank the sender of the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - */ -int PCU_Comm_Sender_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Sender called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_received_from(m->order); - return pcu_msg_received_from(m); -} - -/** \brief Returns true if the current received buffer has been unpacked. - \details This function should be called after a successful PCU_Comm_Listen. - */ -bool PCU_Comm_Unpacked_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Unpacked called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_unpacked(m->order); - return pcu_msg_unpacked(m); -} - -/** \brief Unpacks a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - \a data must point to a block of memory of at least \a size bytes, into - which the next \a size bytes of the current received buffer will be written. - Subsequent calls will begin unpacking where this call left off, - so that the entire received buffer can be unpacked by a sequence of calls to - this function. - Users must ensure that there remain \a size bytes to be unpacked, - PCU_Comm_Unpacked can help with this. - */ -int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Unpack called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - memcpy(data,pcu_order_unpack(m->order,size),size); - else - memcpy(data,pcu_msg_unpack(m,size),size); - return PCU_SUCCESS; -} - -void PCU_Comm_Order_2(pcu_t* pcu, bool on) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Order called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (on && (!m->order)) - m->order = pcu_order_new(); - if ((!on) && m->order) { - pcu_order_free(m->order); - m->order = NULL; - } -} - -/** \brief Blocking barrier over all threads. */ -void PCU_Barrier_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&(pcu->mpi), &(get_msg(pcu)->coll)); -} - -/** \brief Performs an Allreduce sum of double arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n doubles. - After this call, p[i] will contain the sum of all p[i]'s - given by each rank. - */ -void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_doubles,p,n*sizeof(double)); -} - -double PCU_Add_Double_2(pcu_t* pcu, double x) -{ - double a[1]; - a[0] = x; - PCU_Add_Doubles_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of double arrays. - */ -void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_doubles,p,n*sizeof(double)); -} - -double PCU_Min_Double_2(pcu_t* pcu, double x) -{ - double a[1]; - a[0] = x; - PCU_Min_Doubles_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of double arrays. - */ -void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_doubles,p,n*sizeof(double)); -} - -double PCU_Max_Double_2(pcu_t* pcu, double x) -{ - double a[1]; - a[0] = x; - PCU_Max_Doubles_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of integers - */ -void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); -} - -int PCU_Add_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Add_Ints_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of long integers - */ -void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); -} - -long PCU_Add_Long_2(pcu_t* pcu, long x) -{ - long a[1]; - a[0] = x; - PCU_Add_Longs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of size_t unsigned integers - */ -void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Add_SizeTs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of size_t unsigned integers - */ -void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Min_SizeTs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of size_t unsigned integers - */ -void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Max_SizeTs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an exclusive prefix sum of integer arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n integers. - After this call, p[i] will contain the sum of all p[i]'s - given by ranks lower than the calling rank. - */ -void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Exscan_Ints called before Comm_Init"); - int* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); -} - -int PCU_Exscan_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Exscan_Ints_2(pcu, a, 1); - return a[0]; -} - -/** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Exscan_Longs called before Comm_Init"); - long* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); -} - -long PCU_Exscan_Long_2(pcu_t* pcu, long x) -{ - long a[1]; - a[0] = x; - PCU_Exscan_Longs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of int arrays. - */ -void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_ints,p,n*sizeof(int)); -} - -int PCU_Min_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Min_Ints_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of int arrays. - */ -void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_ints,p,n*sizeof(int)); -} - -int PCU_Max_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Max_Ints_2(pcu, a, 1); - return a[0]; -} -/** \brief Performs an Allreduce maximum of long arrays. - */ -void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_longs,p,n*sizeof(long)); -} - -long PCU_Max_Long_2(pcu_t* pcu, long x) -{ - long a[1]; - a[0] = x; - PCU_Max_Longs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs a parallel logical OR reduction - */ -int PCU_Or_2(pcu_t* pcu, int c) -{ - return PCU_Max_Int_2(pcu, c); -} - -/** \brief Performs a parallel logical AND reduction - */ -int PCU_And_2(pcu_t* pcu, int c) -{ - return PCU_Min_Int_2(pcu, c); -} - -/** \brief Returns the unique rank of the calling process. - */ -int PCU_Proc_Self_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(&(pcu->mpi)); -} - -/** \brief Returns the number of processes. - */ -int PCU_Proc_Peers_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(&(pcu->mpi)); -} - -/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. - */ -int PCU_Comm_Rank_2(pcu_t* pcu, int* rank) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(&(pcu->mpi)); - return PCU_SUCCESS; -} - -/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size_2(pcu_t* pcu, int* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(&(pcu->mpi)); - return PCU_SUCCESS; -} - -/** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized_2(pcu_t* pcu) -{ - return pcu->state == pcu_state_init; -} - -/** \brief Deprecated, see PCU_Comm_Begin. - */ -int PCU_Comm_Start_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(&(pcu->mpi),get_msg(pcu)); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. - \details Returns the size of the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) - reel_fail("Invalid rank in Comm_Packed"); - *size = pcu_msg_packed(get_msg(pcu),to_rank); - return PCU_SUCCESS; -} - -/** \brief Packs a message to be sent to \a to_rank. - \details This function packs a message into the buffer being sent - to \a to_rank. - Messages packed by this function can be received using the function - PCU_Comm_Read. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - If this function is used, PCU_Comm_Pack should not be used. - */ -int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) - reel_fail("Invalid rank in Comm_Write"); - pcu_msg* msg = get_msg(pcu); - PCU_MSG_PACK(msg,to_rank,size); - memcpy(pcu_msg_pack(msg,to_rank,size),data,size); - return PCU_SUCCESS; -} - -/** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive_2(pcu_t* pcu) -{ - while (PCU_Comm_Unpacked_2(pcu)) - if (!PCU_Comm_Listen_2(pcu)) - return false; - return true; -} - -/** \brief Receives a message for this communication phase. - \details This function tries to receive a message packed by - PCU_Comm_Write. - If a the communication phase is over and there are no more - messages to receive, this function returns false. - Otherwise, * \a from_rank will be the rank which sent the message, - * \a data will point to the start of the message data, and - * \a size will be the number of bytes of message data. - If this function is used, PCU_Comm_Receive should not be used. - Note that the address * \a data points into a PCU buffer, so - it is strongly recommended that this data be read and not modified. - */ -bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Read called before Comm_Init"); - if (!PCU_Comm_Receive_2(pcu)) - return false; - *from_rank = PCU_Comm_Sender_2(pcu); - PCU_Comm_Unpack_2(pcu, size, sizeof(*size)); - *data = PCU_Comm_Extract_2(pcu, *size); - return true; -} - -static void safe_mkdir(const char* path, mode_t mode) -{ - int err; - errno = 0; - err = mkdir(path, mode); - if (err != 0 && errno != EEXIST) - reel_fail("PCU: could not create directory \"%s\"\n", path); -} - -static void append(char* s, size_t size, const char* format, ...) -{ - int len = strlen(s); - va_list ap; - va_start(ap, format); - vsnprintf(s + len, size - len, format, ap); - va_end(ap); -} - - -void PCU_Debug_Open_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Debug_Open called before Comm_Init"); - - const int fanout = 2048; - const int bufsize = 1024; - char* path = noto_malloc(bufsize); - path[0] = '\0'; - if (PCU_Comm_Peers_2(pcu) > fanout) { - mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - strcpy(path, "debug/"); - safe_mkdir(path, dir_perm); - int self = PCU_Comm_Self_2(pcu); - append(path, bufsize, "%d/", self / fanout); - if (self % fanout == 0) - safe_mkdir(path, dir_perm); - PCU_Barrier_2(pcu); - } - - append(path,bufsize, "%s", "debug"); - pcu_msg* msg = get_msg(pcu); - if ( ! msg->file) - msg->file = pcu_open_parallel(path,"txt"); - noto_free(path); -} - -void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list ap) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Debug_Print called before Comm_Init"); - pcu_msg* msg = get_msg(pcu); - if ( ! msg->file) - return; //Print is a no-op if no file is open - vfprintf(msg->file,format,ap); - fflush(msg->file); -} -/** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) -{ - va_list arglist; - va_start(arglist,format); - PCU_Debug_Printv_2(pcu, format, arglist); - va_end(arglist); -} - -/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From_2(pcu_t* pcu, int* from_rank) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_From called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - *from_rank = pcu_order_received_from(m->order); - else - *from_rank = pcu_msg_received_from(m); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the bytes in the current received buffer - \details This function should be called after a successful PCU_Comm_Receive. - The size returned will be the total received size regardless of how - much unpacking has been done. - */ -int PCU_Comm_Received_2(pcu_t* pcu, size_t* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Received called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - *size = pcu_order_received_size(m->order); - else - *size = pcu_msg_received_size(m); - return PCU_SUCCESS; -} - -/** \brief Extracts a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Receive. - The next \a size bytes of the current received buffer are unpacked, - and an internal pointer to that data is returned. - The returned pointer must not be freed by the user. - */ -void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Extract called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_unpack(m->order,size); - return pcu_msg_unpack(m,size); -} - -/** \brief Reinitializes PCU with a new MPI communicator. - \details All of PCU's logic is based off two duplicates - of this communicator, so you can safely get PCU to act - on sub-groups of processes using this function. - This call should be collective over all processes - in the previous communicator. This is a very heavy weight function - and should be used sparingly. - */ -void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Switch_Comm called before Comm_Init"); - int result; - MPI_Comm_compare(new_comm, pcu->mpi.original_comm, &result); - if (result != MPI_IDENT) { - pcu_mpi_finalize(&(pcu->mpi)); - pcu_mpi_init(new_comm,&(pcu->mpi)); - } -} - -/** \brief Return the current MPI communicator - \details Returns the communicator given to the - most recent PCU_Switch_Comm call, or MPI_COMM_WORLD - otherwise. - */ -MPI_Comm PCU_Get_Comm_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Get_Comm called before Comm_Init"); - return pcu->mpi.original_comm; -} - -/** \brief Return the time in seconds since some time in the past - */ -double PCU_Time_2(pcu_t* pcu) -{ - // silence warning - (void)pcu; - return MPI_Wtime(); -} - -void PCU_Protect_2(pcu_t* pcu) -{ - // silence warning - (void)pcu; - reel_protect(); -} diff --git a/pcu/pcu_buffer.h b/pcu/pcu_buffer.h index e8ff30d22..7695eacee 100644 --- a/pcu/pcu_buffer.h +++ b/pcu/pcu_buffer.h @@ -12,6 +12,9 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif typedef struct { @@ -28,5 +31,8 @@ void* pcu_walk_buffer(pcu_buffer* b, size_t size); bool pcu_buffer_walked(pcu_buffer* b); void pcu_resize_buffer(pcu_buffer* b, size_t size); void pcu_set_buffer(pcu_buffer* b, void* p, size_t size); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_byteorder.h b/pcu/pcu_byteorder.h index 6621b01cb..7ca91c9dd 100644 --- a/pcu/pcu_byteorder.h +++ b/pcu/pcu_byteorder.h @@ -6,6 +6,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + #if CHAR_BIT != 8 #error "unsupported char size" #endif @@ -22,4 +26,8 @@ static const union { #define PCU_HOST_ORDER (pcu_host_order.value) +#ifdef __cplusplus +} +#endif + #endif diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index a3ad4e497..af888c048 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -33,6 +33,7 @@ void pcu_merge_assign(void* local, void* incoming, size_t size) memcpy(local,incoming,size); } +// TODO remove once struct fully inplace void pcu_add_doubles(void* local, void* incoming, size_t size) { double* a = local; diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index 9c89d64f9..aaaf5a6ff 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -12,6 +12,10 @@ #include "pcu_mpi.h" +#ifdef __cplusplus +extern "C" { +#endif + /* The PCU Collectives system (pcu_coll for short) implements non-blocking collective operations based loosely on binary-tree or binomial communication patterns. @@ -88,4 +92,7 @@ void pcu_begin_barrier(pcu_mpi_t*,pcu_coll* c); bool pcu_barrier_done(pcu_mpi_t*, pcu_coll* c); void pcu_barrier(pcu_mpi_t*, pcu_coll* c); +#ifdef __cplusplus +} +#endif #endif //PCU_COLL_H diff --git a/pcu/pcu_defines.h b/pcu/pcu_defines.h new file mode 100644 index 000000000..c39152d16 --- /dev/null +++ b/pcu/pcu_defines.h @@ -0,0 +1,13 @@ +#ifndef SCOREC_PCU_PCU_DEFINES_H +#define SCOREC_PCU_PCU_DEFINES_H + +#define PCU_SUCCESS 0 +#define PCU_FAILURE -1 +#ifdef __GNUC__ +#define PCU_FORMAT_ATTRIBUTE(...) \ + __attribute__((format(printf, ##__VA_ARGS__))); +#else +#define PCU_FORMAT_ATTRIBUTE(format, ...) +#endif + +#endif // SCOREC_PCU_PCU_DEFINES_H diff --git a/pcu/pcu_mem.c b/pcu/pcu_mem.c index 8ac2345c1..bf060b104 100644 --- a/pcu/pcu_mem.c +++ b/pcu/pcu_mem.c @@ -7,7 +7,16 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include +#include +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif #if defined(__APPLE__) @@ -35,7 +44,7 @@ #endif -double PCU_GetMem() { +double pcu_get_mem() { const double M = 1024*1024; #if defined(__APPLE__) bool resident = true; diff --git a/pcu/pcu_mem.h b/pcu/pcu_mem.h new file mode 100644 index 000000000..50f7419b6 --- /dev/null +++ b/pcu/pcu_mem.h @@ -0,0 +1,13 @@ +#ifndef SCOREC_PCU_PCU_MEM_H +#define SCOREC_PCU_PCU_MEM_H + +#ifdef __cplusplus +extern "C" { +#endif + +double pcu_get_mem(); + +#ifdef __cplusplus +} +#endif +#endif // SCOREC_PCU_PCU_MEM_H diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index fb55ea275..970c8a4f2 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -13,6 +13,10 @@ #include "pcu_buffer.h" #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { pcu_buffer buffer; @@ -23,14 +27,15 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); -typedef struct +struct pcu_mpi_struct { MPI_Comm original_comm; MPI_Comm user_comm; MPI_Comm coll_comm; int rank; int size; -} pcu_mpi_t; +}; +typedef struct pcu_mpi_struct pcu_mpi_t; int pcu_mpi_size(const pcu_mpi_t*); int pcu_mpi_rank(const pcu_mpi_t*); @@ -40,4 +45,7 @@ bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); void pcu_mpi_finalize(pcu_mpi_t* mpi); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_msg.h b/pcu/pcu_msg.h index 0289d0d16..ba7595ff6 100644 --- a/pcu/pcu_msg.h +++ b/pcu/pcu_msg.h @@ -14,6 +14,10 @@ #include "pcu_aa.h" #include "pcu_io.h" +#ifdef __cplusplus +extern "C" { +#endif + /* the PCU Messenger (pcu_msg for short) system implements a non-blocking Bulk Synchronous Parallel communication model it is based on PCU non-blocking Collectives and pcu_mpi, @@ -60,4 +64,7 @@ int pcu_msg_received_from(pcu_msg* m); size_t pcu_msg_received_size(pcu_msg* m); void pcu_free_msg(pcu_msg* m); +#ifdef __cplusplus +} +#endif #endif //PCU_MSG_H diff --git a/pcu/pcu_order.h b/pcu/pcu_order.h index 26e144885..1aa870edc 100644 --- a/pcu/pcu_order.h +++ b/pcu/pcu_order.h @@ -13,6 +13,9 @@ #include #include "pcu_msg.h" +#ifdef __cplusplus +extern "C" { +#endif typedef struct pcu_order_struct* pcu_order; pcu_order pcu_order_new(void); @@ -23,4 +26,7 @@ bool pcu_order_unpacked(pcu_order o); int pcu_order_received_from(pcu_order o); size_t pcu_order_received_size(pcu_order o); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index d6dd08dae..7505aa7a4 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2011 Scientific Computation Research Center, + Copyright 2011 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -13,13 +13,18 @@ #include "pcu_mpi.h" #include +#ifdef __cplusplus +extern "C" { +#endif +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t *mpi); +void pcu_pmpi_finalize(pcu_mpi_t *m); +int pcu_pmpi_size(const pcu_mpi_t *self); +int pcu_pmpi_rank(const pcu_mpi_t *self); +void pcu_pmpi_send(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +bool pcu_pmpi_receive(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +bool pcu_pmpi_done(const pcu_mpi_t *, pcu_message *m); -void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* mpi); -void pcu_pmpi_finalize(pcu_mpi_t* m); -int pcu_pmpi_size(const pcu_mpi_t* self); -int pcu_pmpi_rank(const pcu_mpi_t* self); -void pcu_pmpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_done(const pcu_mpi_t*, pcu_message* m); - +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_util.h b/pcu/pcu_util.h index 63aa49a30..e4a248408 100644 --- a/pcu/pcu_util.h +++ b/pcu/pcu_util.h @@ -22,17 +22,32 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); } /* extern "C" */ #endif -#define PCU_ALWAYS_ASSERT(cond) \ - do { \ - if (! (cond)) { \ - char omsg[2048]; \ - sprintf(omsg, "%s failed at %s + %d \n", \ - #cond, __FILE__, __LINE__); \ - PCU_Assert_Fail(omsg); \ - } \ - } while (0) +#define PCU_DO_PRAGMA_(x) _Pragma (#x) +#define PCU_DO_PRAGMA(x) PCU_DO_PRAGMA_(x) + +#define PCU_IGNORE_DIAGNOSTIC_START(x) \ + PCU_DO_PRAGMA(GCC diagnostic push) \ + PCU_DO_PRAGMA(GCC diagnostic ignored #x) \ + PCU_DO_PRAGMA(clang diagnostic push) \ + PCU_DO_PRAGMA(clang diagnostic ignored #x) + +#define PCU_IGNORE_DIAGNOSTIC_END \ + PCU_DO_PRAGMA(GCC diagnostic pop) \ + PCU_DO_PRAGMA(clang diagnostic pop) +#define PCU_ALWAYS_ASSERT(cond) \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ + do { \ + if (! (cond)) { \ + char omsg[2048]; \ + sprintf(omsg, "%s failed at %s + %d \n", \ + #cond, __FILE__, __LINE__); \ + PCU_Assert_Fail(omsg); \ + } \ + } while (0) \ + PCU_IGNORE_DIAGNOSTIC_END #define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ do { \ if (! (cond)) { \ char omsg[2048]; \ @@ -40,7 +55,8 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); #cond, __FILE__, __LINE__, msg); \ PCU_Assert_Fail(omsg); \ } \ - } while(0) + } while(0) \ + PCU_IGNORE_DIAGNOSTIC_END #ifdef NDEBUG #define PCU_DEBUG_ASSERT(cond) diff --git a/pcu/pkg_tribits.cmake b/pcu/pkg_tribits.cmake index 0c43fc64a..ce6e88e92 100644 --- a/pcu/pkg_tribits.cmake +++ b/pcu/pkg_tribits.cmake @@ -33,27 +33,28 @@ include_directories("${PROJECT_BINARY_DIR}") #Sources & Headers set(SOURCES - pcu.c - pcu_aa.c - pcu_coll.c - pcu_io.c - pcu_buffer.c - pcu_mpi.c - pcu_msg.c - pcu_order.c - pcu_pmpi.c - pcu_util.c - noto/noto_malloc.c - reel/reel.c -) + pcu.cc + pcu_aa.c + pcu_coll.c + pcu_io.c + pcu_buffer.c + pcu_mem.c + pcu_mpi.c + pcu_msg.c + pcu_order.c + pcu_pmpi.c + pcu_util.c + noto/noto_malloc.c + reel/reel.c + PCUObj.cc) set(HEADERS - PCU.h - pcu_io.h - pcu_util.h - noto/noto_malloc.h - reel/reel.h) - + PCU.h + pcu_io.h + pcu_util.h + reel/reel.h + pcu_defines.h + ) tribits_add_library( pcu HEADERS ${HEADERS} From eed5a0cd80644b578cc550b65ab3e5d45a3fd502 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 02:14:49 -0400 Subject: [PATCH 011/141] remove unused functions --- pcu/pcu_coll.c | 100 ------------------------------------------------- pcu/pcu_coll.h | 11 ------ 2 files changed, 111 deletions(-) diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index af888c048..01c89445f 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -33,106 +33,6 @@ void pcu_merge_assign(void* local, void* incoming, size_t size) memcpy(local,incoming,size); } -// TODO remove once struct fully inplace -void pcu_add_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_max_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_min_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_add_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_min_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_max_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_max_longs(void* local, void* incoming, size_t size) -{ - long* a = local; - long* b= incoming; - size_t n = size/sizeof(long); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_min_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_max_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_add_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_add_longs(void* local, void* incoming, size_t size) -{ - long* a = local; - long* b= incoming; - size_t n = size/sizeof(long); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - /* initiates non-blocking calls for this communication step */ static void begin_coll_step(pcu_mpi_t* mpi, pcu_coll* c) diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index aaaf5a6ff..f69bf7a08 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -31,17 +31,6 @@ extern "C" { typedef void pcu_merge(void* local, void* incoming, size_t size); void pcu_merge_assign(void* local, void* incoming, size_t size); -void pcu_add_doubles(void* local, void* incoming, size_t size); -void pcu_max_doubles(void* local, void* incoming, size_t size); -void pcu_min_doubles(void* local, void* incoming, size_t size); -void pcu_add_ints(void* local, void* incoming, size_t size); -void pcu_min_ints(void* local, void* incoming, size_t size); -void pcu_max_ints(void* local, void* incoming, size_t size); -void pcu_max_longs(void* local, void* incoming, size_t size); -void pcu_add_longs(void* local, void* incoming, size_t size); -void pcu_add_sizets(void* local, void* incoming, size_t size); -void pcu_min_sizets(void* local, void* incoming, size_t size); -void pcu_max_sizets(void* local, void* incoming, size_t size); /* Enumerated actions that a rank takes during one step of the communication pattern */ From f0b3a28dfef5f302813574932f123833791c897a Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 02:33:40 -0400 Subject: [PATCH 012/141] Fix gcc compilation --- pcu/PCUObj.cc | 4 ++++ pcu/pcu.cc | 3 ++- pcu/pcu_util.h | 30 +++++++++++++++++++++--------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index a6a3f9b76..d0f6f80bc 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -7,6 +7,10 @@ #include "reel.h" #include #include /*using POSIX mkdir call for SMB "foo/" path*/ +#include +#include +#include +#include namespace pcu { int PCU::Peers() const noexcept { return pcu_mpi_size(mpi_); } diff --git a/pcu/pcu.cc b/pcu/pcu.cc index 3a0337812..aeb1f6da1 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -411,7 +411,8 @@ int PCU_Proc_Peers(void) { int PCU_Comm_Rank(int *rank) { if (global_pcu == nullptr) reel_fail("Comm_Rank called before Comm_Init"); - return global_pcu->Self(); + *rank = global_pcu->Self(); + return PCU_SUCCESS; } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ diff --git a/pcu/pcu_util.h b/pcu/pcu_util.h index e4a248408..8122599ad 100644 --- a/pcu/pcu_util.h +++ b/pcu/pcu_util.h @@ -25,38 +25,50 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); #define PCU_DO_PRAGMA_(x) _Pragma (#x) #define PCU_DO_PRAGMA(x) PCU_DO_PRAGMA_(x) +#if defined(__GNUC__) + #define PCU_IGNORE_DIAGNOSTIC_START(x) \ PCU_DO_PRAGMA(GCC diagnostic push) \ - PCU_DO_PRAGMA(GCC diagnostic ignored #x) \ + PCU_DO_PRAGMA(GCC diagnostic ignored #x) +#define PCU_IGNORE_DIAGNOSTIC_END \ + PCU_DO_PRAGMA(GCC diagnostic pop) + +#elif defined(__clang__) + +#define PCU_IGNORE_DIAGNOSTIC_START(x) \ PCU_DO_PRAGMA(clang diagnostic push) \ PCU_DO_PRAGMA(clang diagnostic ignored #x) - #define PCU_IGNORE_DIAGNOSTIC_END \ - PCU_DO_PRAGMA(GCC diagnostic pop) \ PCU_DO_PRAGMA(clang diagnostic pop) +#else +#define PCU_IGNORE_DIAGNOSTIC_START(x) +#define PCU_IGNORE_DIAGNOSTIC_END +#endif + + #define PCU_ALWAYS_ASSERT(cond) \ - PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ if (! (cond)) { \ char omsg[2048]; \ sprintf(omsg, "%s failed at %s + %d \n", \ #cond, __FILE__, __LINE__); \ PCU_Assert_Fail(omsg); \ } \ - } while (0) \ - PCU_IGNORE_DIAGNOSTIC_END + PCU_IGNORE_DIAGNOSTIC_END \ + } while (0) #define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ - PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations)\ if (! (cond)) { \ char omsg[2048]; \ sprintf(omsg, "%s failed at %s + %d \n %s", \ #cond, __FILE__, __LINE__, msg); \ PCU_Assert_Fail(omsg); \ } \ - } while(0) \ - PCU_IGNORE_DIAGNOSTIC_END + PCU_IGNORE_DIAGNOSTIC_END \ + } while(0) #ifdef NDEBUG #define PCU_DEBUG_ASSERT(cond) From 0e7362f25c2fb2a9a76a85a24f4231008433303f Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 20:28:17 -0400 Subject: [PATCH 013/141] fix bug --- pcu/PCUObj.cc | 20 ++++++++++++++++---- pcu/PCUObj.h | 17 ++++++++++++++--- pcu/pcu.cc | 7 ++----- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index d0f6f80bc..40fe39ee5 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -31,7 +31,7 @@ int PCU::Send() noexcept { pcu_msg_send(mpi_, msg_); return PCU_SUCCESS; } -bool PCU::Receive(void) noexcept { +bool PCU::Receive() noexcept { while (Unpacked()) if (!Listen()) return false; @@ -149,9 +149,9 @@ void PCU::DebugOpen() noexcept { noto_free(path); } -double GetMem(void) noexcept { return pcu_get_mem(); } -void Protect(void) noexcept { reel_protect(); } -double Time(void) noexcept { return MPI_Wtime(); } +double GetMem() noexcept { return pcu_get_mem(); } +void Protect() noexcept { reel_protect(); } +double Time() noexcept { return MPI_Wtime(); } void PCU::DebugPrint(const char *format, ...) noexcept { va_list args; @@ -176,7 +176,9 @@ PCU::PCU(MPI_Comm comm) { Order(true); } PCU::~PCU() noexcept { + pcu_mpi_finalize(mpi_); delete mpi_; + pcu_free_msg(msg_); delete msg_; } PCU::PCU(PCU &&other) noexcept { @@ -190,6 +192,16 @@ PCU &PCU::operator=(PCU && other) noexcept { } MPI_Comm PCU::GetMPIComm() const noexcept { return mpi_->original_comm; } +MPI_Comm PCU::SwitchMPIComm(MPI_Comm newcomm) noexcept { + if(newcomm == mpi_->original_comm) { + return mpi_->original_comm; + } + auto original_comm = mpi_->original_comm; + pcu_mpi_finalize(mpi_); + pcu_mpi_init(newcomm, mpi_); + return original_comm; +} + /* template implementations */ template void PCU::Add(T *p, size_t n) noexcept { pcu_allreduce( diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index 9584f5b6c..aea6f263c 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -69,16 +69,27 @@ class PCU { /* Debug functions */ void DebugOpen() noexcept; + MPI_Comm SwitchMPIComm(MPI_Comm) noexcept; + + //struct MPIComms { + // MPI_Comm original; + // MPI_Comm user; + // MPI_Comm coll; + //}; + // takes ownership of newcomms.user & newcomms.coll + // user responsibility to free returned user/coll comm + //MPIComms SwitchMPIComms(MPIComms& newcomms) noexcept; + private: pcu_msg_struct *msg_; pcu_mpi_struct *mpi_; }; /*stack trace helpers using GNU/Linux*/ -void Protect(void) noexcept; +void Protect() noexcept; /*Memory usage*/ -[[nodiscard]] double GetMem(void) noexcept; +[[nodiscard]] double GetMem() noexcept; /*MPI_Wtime() equivalent*/ -[[nodiscard]] double Time(void) noexcept; +[[nodiscard]] double Time() noexcept; /* explicit instantiations of template functions */ #define PCU_EXPL_INST_DECL(T) \ diff --git a/pcu/pcu.cc b/pcu/pcu.cc index aeb1f6da1..0368ff6ff 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -457,7 +457,7 @@ int PCU_Comm_Packed(int to_rank, size_t *size) { int PCU_Comm_Write(int to_rank, const void *data, size_t size) { if (global_pcu == nullptr) reel_fail("Comm_Write called before Comm_Init"); - return global_pcu->Pack(to_rank, data, size); + return global_pcu->Write(to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ @@ -542,10 +542,7 @@ void *PCU_Comm_Extract(size_t size) { void PCU_Switch_Comm(MPI_Comm new_comm) { if (global_pcu == nullptr) reel_fail("Switch_Comm called before Comm_Init"); - if (new_comm != global_pcu->GetMPIComm()) { - delete global_pcu; - global_pcu = new pcu::PCU(new_comm); - } + global_pcu->SwitchMPIComm(new_comm); } /** \brief Return the current MPI communicator From 0e0c80ecb3ce43ef462e2e8657bbdc319e26bca6 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Fri, 31 Mar 2023 01:24:21 -0400 Subject: [PATCH 014/141] remove unnecessary whitespace --- pcu/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index fd5055415..6314df3c2 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,7 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.cc + pcu.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -27,7 +27,7 @@ set(SOURCES # Package headers set(HEADERS PCU.h - pcu_io.h + pcu_io.h pcu_util.h reel/reel.h pcu_defines.h From 07ab3646dccae7eab5b6eff642e7041049e6aaec Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Fri, 31 Mar 2023 01:26:35 -0400 Subject: [PATCH 015/141] bump version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7648e54d3..74ef7b8c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ endif() # This is the top level CMake file for the SCOREC build cmake_minimum_required(VERSION 3.0) -project(SCOREC VERSION 2.2.7 LANGUAGES CXX C) +project(SCOREC VERSION 2.3.0 LANGUAGES CXX C) include(cmake/bob.cmake) include(cmake/xsdk.cmake) From 80c829d4f66a2ea2b8d9f30b8b3d48176e309ae5 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Tue, 28 Mar 2023 04:14:00 -0400 Subject: [PATCH 016/141] Add state to PCU mpi interface. This commit adds a MPI state that's separate from the global variables to the internal pcu_mpi interfaces. The user level PCU interface remains unchanged. However, this change lays the groundwork for PCU2 that will not make use of global state. This is important for some use cases like working around the spectrum-mpi comm_dup bug. The removal of global state also improves testability and modularity of this code. --- pcu/pcu.c | 75 ++++++++++---------- pcu/pcu_coll.c | 179 ++++++++++++++++++++++++------------------------ pcu/pcu_coll.h | 30 ++++---- pcu/pcu_io.c | 13 ++-- pcu/pcu_mpi.c | 53 +++++++------- pcu/pcu_mpi.h | 42 ++++++++---- pcu/pcu_msg.c | 39 +++++------ pcu/pcu_msg.h | 6 +- pcu/pcu_order.c | 8 +-- pcu/pcu_order.h | 2 +- pcu/pcu_pmpi.c | 81 +++++++++++----------- pcu/pcu_pmpi.h | 24 ++----- 12 files changed, 280 insertions(+), 272 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index b6c30f901..afc705504 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -33,7 +33,6 @@ #include #include "PCU.h" #include "pcu_msg.h" -#include "pcu_pmpi.h" #include "pcu_order.h" #include "noto_malloc.h" #include "reel.h" @@ -46,6 +45,7 @@ enum state { uninit, init }; static enum state global_state = uninit; static pcu_msg global_pmsg; +static pcu_mpi_t* global_mpi; static pcu_msg* get_msg() { @@ -61,8 +61,7 @@ int PCU_Comm_Init(void) { if (global_state != uninit) reel_fail("nested calls to Comm_Init"); - pcu_pmpi_init(MPI_COMM_WORLD); - pcu_set_mpi(&pcu_pmpi); + global_mpi = pcu_mpi_init(MPI_COMM_WORLD); pcu_make_msg(&global_pmsg); global_state = init; /* turn ordering on by default, call @@ -83,7 +82,7 @@ int PCU_Comm_Free(void) if (global_pmsg.order) pcu_order_free(global_pmsg.order); pcu_free_msg(&global_pmsg); - pcu_pmpi_finalize(); + pcu_mpi_finalize(&global_mpi); global_state = uninit; return PCU_SUCCESS; } @@ -101,7 +100,7 @@ int PCU_Comm_Self(void) { if (global_state == uninit) reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(); + return pcu_mpi_rank(global_mpi); } /** \brief Returns the number of threads in the program. @@ -112,7 +111,7 @@ int PCU_Comm_Peers(void) { if (global_state == uninit) reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(); + return pcu_mpi_size(global_mpi); } /** \brief Begins a PCU communication phase. @@ -125,7 +124,7 @@ void PCU_Comm_Begin(void) { if (global_state == uninit) reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(get_msg()); + pcu_msg_start(global_mpi, get_msg()); } /** \brief Packs data to be sent to \a to_rank. @@ -138,7 +137,7 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { if (global_state == uninit) reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) reel_fail("Invalid rank in Comm_Pack"); if ( size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); @@ -159,7 +158,7 @@ int PCU_Comm_Send(void) { if (global_state == uninit) reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(get_msg()); + pcu_msg_send(global_mpi, get_msg()); return PCU_SUCCESS; } @@ -180,8 +179,8 @@ bool PCU_Comm_Listen(void) reel_fail("Comm_Listen called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) - return pcu_order_receive(m->order, m); - return pcu_msg_receive(m); + return pcu_order_receive(global_mpi, m->order, m); + return pcu_msg_receive(global_mpi, m); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -250,7 +249,7 @@ void PCU_Barrier(void) { if (global_state == uninit) reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&(get_msg()->coll)); + pcu_barrier(global_mpi, &(get_msg()->coll)); } /** \brief Performs an Allreduce sum of double arrays. @@ -263,7 +262,7 @@ void PCU_Add_Doubles(double* p, size_t n) { if (global_state == uninit) reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); } double PCU_Add_Double(double x) @@ -280,7 +279,7 @@ void PCU_Min_Doubles(double* p, size_t n) { if (global_state == uninit) reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); } double PCU_Min_Double(double x) @@ -297,7 +296,7 @@ void PCU_Max_Doubles(double* p, size_t n) { if (global_state == uninit) reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); } double PCU_Max_Double(double x) @@ -314,7 +313,7 @@ void PCU_Add_Ints(int* p, size_t n) { if (global_state == uninit) reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); } int PCU_Add_Int(int x) @@ -331,7 +330,7 @@ void PCU_Add_Longs(long* p, size_t n) { if (global_state == uninit) reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); } long PCU_Add_Long(long x) @@ -348,7 +347,7 @@ void PCU_Add_SizeTs(size_t* p, size_t n) { if (global_state == uninit) reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); } size_t PCU_Add_SizeT(size_t x) @@ -364,7 +363,7 @@ size_t PCU_Add_SizeT(size_t x) void PCU_Min_SizeTs(size_t* p, size_t n) { if (global_state == uninit) reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); } size_t PCU_Min_SizeT(size_t x) { @@ -379,7 +378,7 @@ size_t PCU_Min_SizeT(size_t x) { void PCU_Max_SizeTs(size_t* p, size_t n) { if (global_state == uninit) reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); } size_t PCU_Max_SizeT(size_t x) { @@ -403,7 +402,7 @@ void PCU_Exscan_Ints(int* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(&(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -427,7 +426,7 @@ void PCU_Exscan_Longs(long* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(&(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -448,7 +447,7 @@ void PCU_Min_Ints(int* p, size_t n) { if (global_state == uninit) reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); } int PCU_Min_Int(int x) @@ -465,7 +464,7 @@ void PCU_Max_Ints(int* p, size_t n) { if (global_state == uninit) reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); } int PCU_Max_Int(int x) @@ -481,7 +480,7 @@ void PCU_Max_Longs(long* p, size_t n) { if (global_state == uninit) reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); } long PCU_Max_Long(long x) @@ -512,7 +511,7 @@ int PCU_Proc_Self(void) { if (global_state == uninit) reel_fail("Proc_Self called before Comm_Init"); - return pcu_pmpi_rank(); + return pcu_mpi_rank(global_mpi); } /** \brief Returns the number of processes. @@ -521,7 +520,7 @@ int PCU_Proc_Peers(void) { if (global_state == uninit) reel_fail("Proc_Peers called before Comm_Init"); - return pcu_pmpi_size(); + return pcu_mpi_size(global_mpi); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. @@ -530,7 +529,7 @@ int PCU_Comm_Rank(int* rank) { if (global_state == uninit) reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(); + *rank = pcu_mpi_rank(global_mpi); return PCU_SUCCESS; } @@ -539,7 +538,7 @@ int PCU_Comm_Size(int* size) { if (global_state == uninit) reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(); + *size = pcu_mpi_size(global_mpi); return PCU_SUCCESS; } @@ -556,7 +555,7 @@ int PCU_Comm_Start(PCU_Method method) (void)method; //warning silencer if (global_state == uninit) reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(get_msg()); + pcu_msg_start(global_mpi,get_msg()); return PCU_SUCCESS; } @@ -569,7 +568,7 @@ int PCU_Comm_Packed(int to_rank, size_t* size) { if (global_state == uninit) reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) reel_fail("Invalid rank in Comm_Packed"); *size = pcu_msg_packed(get_msg(),to_rank); return PCU_SUCCESS; @@ -588,7 +587,7 @@ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { if (global_state == uninit) reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) reel_fail("Invalid rank in Comm_Write"); pcu_msg* msg = get_msg(); PCU_MSG_PACK(msg,to_rank,size); @@ -741,13 +740,19 @@ void* PCU_Comm_Extract(size_t size) of this communicator, so you can safely get PCU to act on sub-groups of processes using this function. This call should be collective over all processes - in the previous communicator. + in the previous communicator. This is a very heavy weight function + and should be used sparingly. */ void PCU_Switch_Comm(MPI_Comm new_comm) { if (global_state == uninit) reel_fail("Switch_Comm called before Comm_Init"); - pcu_pmpi_switch(new_comm); + int result; + MPI_Comm_compare(new_comm, global_mpi->original_comm, &result); + if (result != MPI_IDENT) { + pcu_mpi_finalize(&global_mpi); + pcu_mpi_init(new_comm); + } } /** \brief Return the current MPI communicator @@ -759,7 +764,7 @@ MPI_Comm PCU_Get_Comm(void) { if (global_state == uninit) reel_fail("Get_Comm called before Comm_Init"); - return pcu_pmpi_comm(); + return global_mpi->original_comm; } /** \brief Return the time in seconds since some time in the past diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 7bdf8a221..39405aa6b 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_coll.h" -#include "pcu_pmpi.h" #include "reel.h" #include @@ -135,31 +134,31 @@ void pcu_add_longs(void* local, void* incoming, size_t size) /* initiates non-blocking calls for this communication step */ -static void begin_coll_step(pcu_coll* c) +static void begin_coll_step(pcu_mpi_t* mpi, pcu_coll* c) { - int action = c->pattern->action(c->bit); + int action = c->pattern->action(mpi, c->bit); if (action == pcu_coll_idle) return; - c->message.peer = c->pattern->peer(c->bit); + c->message.peer = c->pattern->peer(mpi, c->bit); if (action == pcu_coll_send) - pcu_mpi_send(&(c->message),pcu_coll_comm); + pcu_mpi_send(mpi, &(c->message),mpi->coll_comm); } /* tries to complete this communication step. Returns false if communication is not done, otherwise wraps up communication, merges if necessary, and returns true */ -static bool end_coll_step(pcu_coll* c) +static bool end_coll_step(pcu_mpi_t* mpi, pcu_coll* c) { - int action = c->pattern->action(c->bit); + int action = c->pattern->action(mpi, c->bit); if (action == pcu_coll_idle) return true; if (action == pcu_coll_send) - return pcu_mpi_done(&(c->message)); + return pcu_mpi_done(mpi, &(c->message)); pcu_message incoming; pcu_make_message(&incoming); - incoming.peer = c->pattern->peer(c->bit); - if ( ! pcu_mpi_receive(&incoming,pcu_coll_comm)) + incoming.peer = c->pattern->peer(mpi, c->bit); + if ( ! pcu_mpi_receive(mpi, &incoming,mpi->coll_comm)) return false; if (c->message.buffer.size != incoming.buffer.size) reel_fail("PCU unexpected incoming message.\n" @@ -169,7 +168,7 @@ static bool end_coll_step(pcu_coll* c) return true; } -void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m) +void pcu_make_coll(pcu_mpi_t* mpi, pcu_coll* c, pcu_pattern* p, pcu_merge* m) { c->pattern = p; c->merge = m; @@ -194,28 +193,28 @@ void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m) /* begins a non-blocking collective. The collective operation should be set with pcu_make_coll first. data[0..size] is the input/output local data */ -void pcu_begin_coll(pcu_coll* c, void* data, size_t size) +void pcu_begin_coll(pcu_mpi_t* mpi, pcu_coll* c, void* data, size_t size) { pcu_set_buffer(&(c->message.buffer),data,size); - c->bit = c->pattern->begin_bit(); - if (c->pattern->end_bit(c->bit)) + c->bit = c->pattern->begin_bit(mpi); + if (c->pattern->end_bit(mpi, c->bit)) return; - begin_coll_step(c); + begin_coll_step(mpi, c); } /* makes progress on a collective operation started by pcu_begin_coll. returns false if its done. */ -bool pcu_progress_coll(pcu_coll* c) +bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) { - if (c->pattern->end_bit(c->bit)) + if (c->pattern->end_bit(mpi, c->bit)) return false; - if (end_coll_step(c)) + if (end_coll_step(mpi, c)) { - c->bit = c->pattern->shift(c->bit); - if (c->pattern->end_bit(c->bit)) + c->bit = c->pattern->shift(mpi, c->bit); + if (c->pattern->end_bit(mpi, c->bit)) return false; - begin_coll_step(c); + begin_coll_step(mpi, c); } return true; } @@ -224,34 +223,34 @@ bool pcu_progress_coll(pcu_coll* c) then odd multiples of 2 into even ones, etc... until rank 0 has all inputs merged */ -static int reduce_begin_bit(void) +static int reduce_begin_bit(pcu_mpi_t*) { return 1; } -static bool reduce_end_bit(int bit) +static bool reduce_end_bit(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (rank==0) - return bit >= pcu_mpi_size(); + return bit >= pcu_mpi_size(mpi); return (bit>>1) & rank; } -static int reduce_peer(int bit) +static int reduce_peer(pcu_mpi_t* mpi, int bit) { - return pcu_mpi_rank() ^ bit; + return pcu_mpi_rank(mpi) ^ bit; } -static int reduce_action(int bit) +static int reduce_action(pcu_mpi_t* mpi, int bit) { - if (reduce_peer(bit) >= pcu_mpi_size()) + if (reduce_peer(mpi, bit) >= pcu_mpi_size(mpi)) return pcu_coll_idle; - if (bit & pcu_mpi_rank()) + if (bit & pcu_mpi_rank(mpi)) return pcu_coll_send; return pcu_coll_recv; } -static int reduce_shift(int bit) +static int reduce_shift(pcu_mpi_t*, int bit) { return bit << 1; } @@ -269,36 +268,36 @@ static pcu_pattern reduce = the pattern runs backwards and send/recv are flipped. */ -static int bcast_begin_bit(void) +static int bcast_begin_bit(pcu_mpi_t* mpi) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (rank == 0) - return 1 << ceil_log2(pcu_mpi_size()); + return 1 << ceil_log2(pcu_mpi_size(mpi)); int bit = 1; while ( ! (bit & rank)) bit <<= 1; return bit; } -static bool bcast_end_bit(int bit) +static bool bcast_end_bit(pcu_mpi_t *, int bit) { return bit == 0; } -static int bcast_peer(int bit) +static int bcast_peer(pcu_mpi_t * mpi, int bit) { - return pcu_mpi_rank() ^ bit; + return pcu_mpi_rank(mpi) ^ bit; } -static int bcast_action(int bit) +static int bcast_action(pcu_mpi_t* mpi, int bit) { - if (bcast_peer(bit) >= pcu_mpi_size()) + if (bcast_peer(mpi, bit) >= pcu_mpi_size(mpi)) return pcu_coll_idle; - if (bit & pcu_mpi_rank()) + if (bit & pcu_mpi_rank(mpi)) return pcu_coll_recv; return pcu_coll_send; } -static int bcast_shift(int bit) +static int bcast_shift(pcu_mpi_t *, int bit) { return bit >> 1; } @@ -319,14 +318,14 @@ static pcu_pattern bcast = "Parallel Prefix (Scan) Algorithms for MPI". */ -static int scan_up_begin_bit(void) +static int scan_up_begin_bit(pcu_mpi_t*) { return 1; } -static bool scan_up_end_bit(int bit) +static bool scan_up_end_bit(pcu_mpi_t* mpi, int bit) { - return bit == (1 << floor_log2(pcu_mpi_size())); + return bit == (1 << floor_log2(pcu_mpi_size(mpi))); } static bool scan_up_could_receive(int rank, int bit) @@ -345,34 +344,34 @@ static int scan_up_receiver_for(int rank, int bit) return rank + bit; } -static int scan_up_action(int bit) +static int scan_up_action(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if ((scan_up_could_receive(rank,bit))&& (0 <= scan_up_sender_for(rank,bit))) return pcu_coll_recv; int receiver = scan_up_receiver_for(rank,bit); - if ((receiver < pcu_mpi_size())&& + if ((receiver < pcu_mpi_size(mpi))&& (scan_up_could_receive(receiver,bit))) return pcu_coll_send; return pcu_coll_idle; } -static int scan_up_peer(int bit) +static int scan_up_peer(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); int sender = scan_up_sender_for(rank,bit); if ((scan_up_could_receive(rank,bit))&& (0 <= sender)) return sender; int receiver = scan_up_receiver_for(rank,bit); - if ((receiver < pcu_mpi_size())&& + if ((receiver < pcu_mpi_size(mpi))&& (scan_up_could_receive(receiver,bit))) return receiver; return -1; } -static int scan_up_shift(int bit) +static int scan_up_shift(pcu_mpi_t*, int bit) { return bit << 1; } @@ -386,12 +385,12 @@ static pcu_pattern scan_up = .shift = scan_up_shift, }; -static int scan_down_begin_bit(void) +static int scan_down_begin_bit(pcu_mpi_t* mpi) { - return 1 << floor_log2(pcu_mpi_size()); + return 1 << floor_log2(pcu_mpi_size(mpi)); } -static bool scan_down_end_bit(int bit) +static bool scan_down_end_bit(pcu_mpi_t*, int bit) { return bit == 1; } @@ -412,11 +411,11 @@ static int scan_down_sender_for(int rank, int bit) return rank - (bit >> 1); } -static int scan_down_action(int bit) +static int scan_down_action(pcu_mpi_t * mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if ((scan_down_could_send(rank,bit))&& - (scan_down_receiver_for(rank,bit) < pcu_mpi_size())) + (scan_down_receiver_for(rank,bit) < pcu_mpi_size(mpi))) return pcu_coll_send; int sender = scan_down_sender_for(rank,bit); if ((0 <= sender)&& @@ -425,13 +424,13 @@ static int scan_down_action(int bit) return pcu_coll_idle; } -static int scan_down_peer(int bit) +static int scan_down_peer(pcu_mpi_t * mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (scan_down_could_send(rank,bit)) { int receiver = scan_down_receiver_for(rank,bit); - if (receiver < pcu_mpi_size()) + if (receiver < pcu_mpi_size(mpi)) return receiver; } int sender = scan_down_sender_for(rank,bit); @@ -441,7 +440,7 @@ static int scan_down_peer(int bit) return -1; } -static int scan_down_shift(int bit) +static int scan_down_shift(pcu_mpi_t*, int bit) { return bit >> 1; } @@ -455,59 +454,59 @@ static pcu_pattern scan_down = .shift = scan_down_shift, }; -void pcu_reduce(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_reduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_make_coll(c,&reduce,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&reduce,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } -void pcu_bcast(pcu_coll* c, void* data, size_t size) +void pcu_bcast(pcu_mpi_t * mpi, pcu_coll* c, void* data, size_t size) { - pcu_make_coll(c,&bcast,pcu_merge_assign); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&bcast,pcu_merge_assign); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } -void pcu_allreduce(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_allreduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_reduce(c,m,data,size); - pcu_bcast(c,data,size); + pcu_reduce(mpi, c,m,data,size); + pcu_bcast(mpi, c,data,size); } -void pcu_scan(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_scan(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_make_coll(c,&scan_up,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); - pcu_make_coll(c,&scan_down,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&scan_up,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); + pcu_make_coll(mpi, c,&scan_down,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } /* a barrier is just an allreduce of nothing in particular */ -void pcu_begin_barrier(pcu_coll* c) +void pcu_begin_barrier(pcu_mpi_t* mpi, pcu_coll* c) { - pcu_make_coll(c,&reduce,pcu_merge_assign); - pcu_begin_coll(c,NULL,0); + pcu_make_coll(mpi, c,&reduce,pcu_merge_assign); + pcu_begin_coll(mpi, c,NULL,0); } -bool pcu_barrier_done(pcu_coll* c) +bool pcu_barrier_done(pcu_mpi_t* mpi, pcu_coll* c) { if (c->pattern == &reduce) - if ( ! pcu_progress_coll(c)) + if ( ! pcu_progress_coll(mpi, c)) { - pcu_make_coll(c,&bcast,pcu_merge_assign); - pcu_begin_coll(c,c->message.buffer.start,c->message.buffer.size); + pcu_make_coll(mpi, c,&bcast,pcu_merge_assign); + pcu_begin_coll(mpi, c,c->message.buffer.start,c->message.buffer.size); } if (c->pattern == &bcast) - if ( ! pcu_progress_coll(c)) + if ( ! pcu_progress_coll(mpi,c)) return true; return false; } -void pcu_barrier(pcu_coll* c) +void pcu_barrier(pcu_mpi_t* mpi, pcu_coll* c) { - pcu_begin_barrier(c); - while( ! pcu_barrier_done(c)); + pcu_begin_barrier(mpi, c); + while( ! pcu_barrier_done(mpi, c)); } diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index 11f204ee8..9c89d64f9 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -57,11 +57,11 @@ enum */ typedef struct { - int (*begin_bit)(void); //initialize state bit - bool (*end_bit)(int bit); //return true if bit is one past the last - int (*action)(int bit); //return action enum for this step - int (*peer)(int bit); //return the peer to communicate with - int (*shift)(int bit); //shift the bit up or down + int (*begin_bit)(pcu_mpi_t*); //initialize state bit + bool (*end_bit)(pcu_mpi_t*, int bit); //return true if bit is one past the last + int (*action)(pcu_mpi_t*, int bit); //return action enum for this step + int (*peer)(pcu_mpi_t*, int bit); //return the peer to communicate with + int (*shift)(pcu_mpi_t*, int bit); //shift the bit up or down } pcu_pattern; /* The pcu_coll object stores the state of a non-blocking @@ -74,18 +74,18 @@ typedef struct int bit; //pattern's state bit } pcu_coll; -void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m); -void pcu_begin_coll(pcu_coll* c, void* data, size_t size); +void pcu_make_coll(pcu_mpi_t *, pcu_coll* c, pcu_pattern* p, pcu_merge* m); +void pcu_begin_coll(pcu_mpi_t *, pcu_coll* c, void* data, size_t size); //returns false when done -bool pcu_progress_coll(pcu_coll* c); +bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c); -void pcu_reduce(pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_bcast(pcu_coll* c, void* data, size_t size); -void pcu_allreduce(pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_scan(pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_reduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_bcast(pcu_mpi_t*, pcu_coll* c, void* data, size_t size); +void pcu_allreduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_scan(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_begin_barrier(pcu_coll* c); -bool pcu_barrier_done(pcu_coll* c); -void pcu_barrier(pcu_coll* c); +void pcu_begin_barrier(pcu_mpi_t*,pcu_coll* c); +bool pcu_barrier_done(pcu_mpi_t*, pcu_coll* c); +void pcu_barrier(pcu_mpi_t*, pcu_coll* c); #endif //PCU_COLL_H diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index a706f6155..a89f366bd 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,16 +8,16 @@ *******************************************************************************/ #include "pcu_io.h" +#include "PCU.h" #include "noto_malloc.h" +#include "pcu_buffer.h" +#include "pcu_util.h" #include "reel.h" -#include "pcu_mpi.h" -#include "PCU.h" +#include #include -#include #include -#include "pcu_util.h" +#include #include -#include #ifdef PCU_BZIP #include @@ -361,7 +361,8 @@ FILE* pcu_open_parallel(const char* prefix, const char* ext) static const size_t max_rank_chars = 10; size_t path_size = strlen(prefix) + max_rank_chars + strlen(ext) + 1; char* path = noto_malloc(path_size); - int rank = pcu_mpi_rank(); + int rank; + PCU_Comm_Rank(&rank); snprintf(path,path_size,"%s%d.%s",prefix,rank,ext); FILE* file = fopen(path, "w"); noto_free(path); diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index 75009d277..c5dea8baa 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -9,8 +9,7 @@ *******************************************************************************/ #include "pcu_mpi.h" #include "pcu_util.h" - -static pcu_mpi* global_mpi; +#include "pcu_pmpi.h" void pcu_make_message(pcu_message* m) { @@ -22,47 +21,49 @@ void pcu_free_message(pcu_message* m) pcu_free_buffer(&(m->buffer)); } -void pcu_set_mpi(pcu_mpi* m) +int pcu_mpi_size(pcu_mpi_t* self) { - global_mpi = m; + return pcu_pmpi_size(self); } -pcu_mpi* pcu_get_mpi(void) +int pcu_mpi_rank(pcu_mpi_t* self) { - return global_mpi; + return pcu_pmpi_rank(self); } -int pcu_mpi_size(void) -{ - return global_mpi->size(); -} - -int pcu_mpi_rank(void) -{ - return global_mpi->rank(); -} - -static void check_rank(int rank) +static void check_rank(pcu_mpi_t* self, int rank) { (void)rank; PCU_ALWAYS_ASSERT(0 <= rank); - PCU_ALWAYS_ASSERT(rank < pcu_mpi_size()); + PCU_ALWAYS_ASSERT(rank < pcu_mpi_size(self)); } -void pcu_mpi_send(pcu_message* m, MPI_Comm comm) +void pcu_mpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - check_rank(m->peer); - global_mpi->send(m,comm); + check_rank(self, m->peer); + // verify that the communicator is one of the user or collective communicators + //int user_result, coll_result; + //MPI_Comm_compare(comm, self->user_comm, &user_result); + //MPI_Comm_compare(comm, self->coll_comm, &coll_result); + //PCU_ALWAYS_ASSERT(user_result == MPI_IDENT || coll_result == MPI_IDENT); + PCU_ALWAYS_ASSERT(comm == self->user_comm || comm == self->coll_comm); + pcu_pmpi_send(self, m, comm); } -bool pcu_mpi_done(pcu_message* m) +bool pcu_mpi_done(pcu_mpi_t* self, pcu_message* m) { - return global_mpi->done(m); + return pcu_pmpi_done(self, m); } -bool pcu_mpi_receive(pcu_message* m, MPI_Comm comm) +bool pcu_mpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { if (m->peer != MPI_ANY_SOURCE) - check_rank(m->peer); - return global_mpi->receive(m,comm); + check_rank(self, m->peer); + return pcu_pmpi_receive(self, m, comm); +} +pcu_mpi_t* pcu_mpi_init(MPI_Comm comm) { + return pcu_pmpi_init(comm); +} +void pcu_mpi_finalize(pcu_mpi_t** mpi) { + pcu_pmpi_finalize(mpi); } diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 052920d04..3c2dcc1f9 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -23,21 +23,35 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); +struct pcu_mpi_t_t; typedef struct { - int (*size)(void); - int (*rank)(void); - void (*send)(pcu_message* m, MPI_Comm comm); - bool (*done)(pcu_message* m); - bool (*receive)(pcu_message* m, MPI_Comm comm); -} pcu_mpi; - -void pcu_set_mpi(pcu_mpi* m); -pcu_mpi* pcu_get_mpi(void); -int pcu_mpi_size(void); -int pcu_mpi_rank(void); -void pcu_mpi_send(pcu_message* m, MPI_Comm comm); -bool pcu_mpi_done(pcu_message* m); -bool pcu_mpi_receive(pcu_message* m, MPI_Comm comm); + int (*size)(struct pcu_mpi_t_t*); + int (*rank)(struct pcu_mpi_t_t*); + void (*send)(struct pcu_mpi_t_t *, pcu_message* m, MPI_Comm comm); + bool (*done)(struct pcu_mpi_t_t*, pcu_message* m); + bool (*receive)(struct pcu_mpi_t_t*, pcu_message* m, MPI_Comm comm); + void (*construct)(struct pcu_mpi_t_t*, MPI_Comm comm); + void (*destruct)(struct pcu_mpi_t_t*); +} pcu_mpi_vtable; + +struct pcu_mpi_t_t +{ + pcu_mpi_vtable const* vtable; + MPI_Comm original_comm; + MPI_Comm user_comm; + MPI_Comm coll_comm; + int rank; + int size; +}; +typedef struct pcu_mpi_t_t pcu_mpi_t; + +int pcu_mpi_size(pcu_mpi_t*); +int pcu_mpi_rank(pcu_mpi_t*); +void pcu_mpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_mpi_done(pcu_mpi_t*, pcu_message* m); +bool pcu_mpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +pcu_mpi_t* pcu_mpi_init(MPI_Comm comm); +void pcu_mpi_finalize(pcu_mpi_t**); #endif diff --git a/pcu/pcu_msg.c b/pcu/pcu_msg.c index 2b4705890..538310c30 100644 --- a/pcu/pcu_msg.c +++ b/pcu/pcu_msg.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_msg.h" -#include "pcu_pmpi.h" #include "noto_malloc.h" #include "reel.h" #include @@ -87,14 +86,14 @@ static void free_peers(pcu_aa_tree* t) pcu_make_aa(t); } -void pcu_msg_start(pcu_msg* m) +void pcu_msg_start(pcu_mpi_t* mpi, pcu_msg* m) { if (m->state != idle_state) reel_fail("PCU_Comm_Begin called at the wrong time"); /* this barrier ensures no one starts a new superstep while others are receiving in the past superstep. It is the only blocking call in the pcu_msg system. */ - pcu_barrier(&(m->coll)); + pcu_barrier(mpi, &(m->coll)); m->state = pack_state; } @@ -143,49 +142,49 @@ size_t pcu_msg_packed(pcu_msg* m, int id) return peer->message.buffer.size; } -static void send_peers(pcu_aa_tree t) +static void send_peers(pcu_mpi_t* mpi, pcu_aa_tree t) { if (pcu_aa_empty(t)) return; pcu_msg_peer* peer; peer = (pcu_msg_peer*)t; - pcu_mpi_send(&(peer->message),pcu_user_comm); - send_peers(t->left); - send_peers(t->right); + pcu_mpi_send(mpi, &(peer->message),mpi->user_comm); + send_peers(mpi, t->left); + send_peers(mpi, t->right); } -void pcu_msg_send(pcu_msg* m) +void pcu_msg_send(pcu_mpi_t* mpi, pcu_msg* m) { if (m->state != pack_state) reel_fail("PCU_Comm_Send called at the wrong time"); - send_peers(m->peers); + send_peers(mpi, m->peers); m->state = send_recv_state; } -static bool done_sending_peers(pcu_aa_tree t) +static bool done_sending_peers(pcu_mpi_t* mpi, pcu_aa_tree t) { if (pcu_aa_empty(t)) return true; pcu_msg_peer* peer; peer = (pcu_msg_peer*)t; - return pcu_mpi_done(&(peer->message)) - && done_sending_peers(t->left) - && done_sending_peers(t->right); + return pcu_mpi_done(mpi, &(peer->message)) + && done_sending_peers(mpi, t->left) + && done_sending_peers(mpi, t->right); } -static bool receive_global(pcu_msg* m) +static bool receive_global(pcu_mpi_t* mpi, pcu_msg* m) { m->received.peer = MPI_ANY_SOURCE; - while ( ! pcu_mpi_receive(&(m->received),pcu_user_comm)) + while ( ! pcu_mpi_receive(mpi, &(m->received),mpi->user_comm)) { if (m->state == send_recv_state) - if (done_sending_peers(m->peers)) + if (done_sending_peers(mpi, m->peers)) { - pcu_begin_barrier(&(m->coll)); + pcu_begin_barrier(mpi, &(m->coll)); m->state = recv_state; } if (m->state == recv_state) - if (pcu_barrier_done(&(m->coll))) + if (pcu_barrier_done(mpi, &(m->coll))) return false; } return true; @@ -197,14 +196,14 @@ static void free_comm(pcu_msg* m) pcu_free_message(&(m->received)); } -bool pcu_msg_receive(pcu_msg* m) +bool pcu_msg_receive(pcu_mpi_t* mpi, pcu_msg* m) { if ((m->state != send_recv_state)&& (m->state != recv_state)) reel_fail("PCU_Comm_Receive called at the wrong time"); if ( ! pcu_msg_unpacked(m)) reel_fail("PCU_Comm_Receive called before previous message unpacked"); - if (receive_global(m)) + if (receive_global(mpi, m)) { pcu_begin_buffer(&(m->received.buffer)); return true; diff --git a/pcu/pcu_msg.h b/pcu/pcu_msg.h index 48713e80a..0289d0d16 100644 --- a/pcu/pcu_msg.h +++ b/pcu/pcu_msg.h @@ -45,13 +45,13 @@ struct pcu_msg_struct typedef struct pcu_msg_struct pcu_msg; void pcu_make_msg(pcu_msg* m); -void pcu_msg_start(pcu_msg* b); +void pcu_msg_start(pcu_mpi_t*, pcu_msg* b); void* pcu_msg_pack(pcu_msg* m, int id, size_t size); #define PCU_MSG_PACK(m,id,o) \ memcpy(pcu_msg_pack(m,id,sizeof(o)),&(o),sizeof(o)) size_t pcu_msg_packed(pcu_msg* m, int id); -void pcu_msg_send(pcu_msg* m); -bool pcu_msg_receive(pcu_msg* m); +void pcu_msg_send(pcu_mpi_t *mpi, pcu_msg* m); +bool pcu_msg_receive(pcu_mpi_t* mpi, pcu_msg* m); void* pcu_msg_unpack(pcu_msg* m, size_t size); #define PCU_MSG_UNPACK(m,o) \ memcpy(&(o),pcu_msg_unpack(m,sizeof(o)),sizeof(o)) diff --git a/pcu/pcu_order.c b/pcu/pcu_order.c index 1099e2d80..d0989a056 100644 --- a/pcu/pcu_order.c +++ b/pcu/pcu_order.c @@ -102,10 +102,10 @@ static void fill(pcu_order o, pcu_aa_tree t) fill(o, t->right); } -static void prepare(pcu_order o, pcu_msg* t) +static void prepare(pcu_mpi_t* mpi, pcu_order o, pcu_msg* t) { struct message* m; - while (pcu_msg_receive(t)) { + while (pcu_msg_receive(mpi, t)) { m = take_message(t); pcu_aa_insert(&m->node, &o->tree, message_less); } @@ -117,10 +117,10 @@ static void prepare(pcu_order o, pcu_msg* t) o->ready = true; } -bool pcu_order_receive(pcu_order o, pcu_msg* m) +bool pcu_order_receive(pcu_mpi_t* mpi, pcu_order o, pcu_msg* m) { if (!o->ready) - prepare(o, m); + prepare(mpi, o, m); o->at++; if (o->at == o->count) { dtor_order(o); diff --git a/pcu/pcu_order.h b/pcu/pcu_order.h index b762a1752..26e144885 100644 --- a/pcu/pcu_order.h +++ b/pcu/pcu_order.h @@ -17,7 +17,7 @@ typedef struct pcu_order_struct* pcu_order; pcu_order pcu_order_new(void); void pcu_order_free(pcu_order o); -bool pcu_order_receive(pcu_order o, pcu_msg* m); +bool pcu_order_receive(pcu_mpi_t*, pcu_order o, pcu_msg* m); void* pcu_order_unpack(pcu_order o, size_t size); bool pcu_order_unpacked(pcu_order o); int pcu_order_received_from(pcu_order o); diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 814828a7e..44b53c762 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -13,51 +13,62 @@ #include #include -static int global_size; -static int global_rank; +void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); +bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); +void pcu_pmpi_destruct(pcu_mpi_t* self); +void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm); -MPI_Comm original_comm; -MPI_Comm pcu_user_comm; -MPI_Comm pcu_coll_comm; - -pcu_mpi pcu_pmpi = +static pcu_mpi_vtable pcu_pmpi_vtable = { .size = pcu_pmpi_size, .rank = pcu_pmpi_rank, .send = pcu_pmpi_send, .done = pcu_pmpi_done, - .receive = pcu_pmpi_receive }; + .receive = pcu_pmpi_receive, + .construct = pcu_pmpi_construct, + .destruct = pcu_pmpi_destruct }; -void pcu_pmpi_init(MPI_Comm comm) +pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) { - original_comm = comm; - MPI_Comm_dup(comm,&pcu_user_comm); - MPI_Comm_dup(comm,&pcu_coll_comm); - MPI_Comm_size(comm,&global_size); - MPI_Comm_rank(comm,&global_rank); + pcu_mpi_t* m = (pcu_mpi_t*)malloc(sizeof(pcu_mpi_t)); + pcu_pmpi_construct(m,comm); + return m; } -void pcu_pmpi_finalize(void) +void pcu_pmpi_finalize(pcu_mpi_t** m) { - MPI_Comm_free(&pcu_user_comm); - MPI_Comm_free(&pcu_coll_comm); + (*m)->vtable->destruct(*m); + free(*m); + *m = NULL; +} +void pcu_pmpi_destruct(pcu_mpi_t* self) { + MPI_Comm_free(&(self->user_comm)); + MPI_Comm_free(&(self->coll_comm)); +} +void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { + self->original_comm = comm; + MPI_Comm_dup(comm,&(self->user_comm)); + MPI_Comm_dup(comm,&(self->coll_comm)); + MPI_Comm_size(comm,&(self->size)); + MPI_Comm_rank(comm,&(self->rank)); + self->vtable = &pcu_pmpi_vtable; } -int pcu_pmpi_size(void) +int pcu_pmpi_size(pcu_mpi_t* self) { - return global_size; + return self->size; } -int pcu_pmpi_rank(void) +int pcu_pmpi_rank(pcu_mpi_t* self) { - return global_rank; + return self->rank; } -void pcu_pmpi_send(pcu_message* m, MPI_Comm comm) +void pcu_pmpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - pcu_pmpi_send2(m,0,comm); + pcu_pmpi_send2(self, m,0,comm); } -void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm) +void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { if( m->buffer.size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR PCU message size exceeds INT_MAX... exiting\n"); @@ -73,19 +84,19 @@ void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm) &(m->request)); } -bool pcu_pmpi_done(pcu_message* m) +bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) { int flag; MPI_Test(&(m->request),&flag,MPI_STATUS_IGNORE); return flag; } -bool pcu_pmpi_receive(pcu_message* m, MPI_Comm comm) +bool pcu_pmpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - return pcu_pmpi_receive2(m,0,comm); + return pcu_pmpi_receive2(self, m,0,comm); } -bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm) +bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { MPI_Status status; int flag; @@ -105,16 +116,4 @@ bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm) comm, MPI_STATUS_IGNORE); return true; -} - -void pcu_pmpi_switch(MPI_Comm new_comm) -{ - pcu_pmpi_finalize(); - pcu_pmpi_init(new_comm); -} - -MPI_Comm pcu_pmpi_comm(void) -{ - return original_comm; -} - +} \ No newline at end of file diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index c6707d23e..cdfbbf699 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -14,22 +14,12 @@ #include -void pcu_pmpi_init(MPI_Comm comm); -void pcu_pmpi_finalize(void); -int pcu_pmpi_size(void); -int pcu_pmpi_rank(void); -void pcu_pmpi_send(pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(pcu_message* m, MPI_Comm comm); -void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_done(pcu_message* m); - -void pcu_pmpi_switch(MPI_Comm new_comm); -MPI_Comm pcu_pmpi_comm(void); - -extern pcu_mpi pcu_pmpi; - -extern MPI_Comm pcu_user_comm; -extern MPI_Comm pcu_coll_comm; +pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm); +void pcu_pmpi_finalize(pcu_mpi_t** m); +int pcu_pmpi_size(pcu_mpi_t* self); +int pcu_pmpi_rank(pcu_mpi_t* self); +void pcu_pmpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_done(pcu_mpi_t*, pcu_message* m); #endif From 2f6238c56bffd02156a83158398663ac721885f3 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Tue, 28 Mar 2023 16:01:41 -0400 Subject: [PATCH 017/141] Compile on gcc+fix bug --- pcu/pcu.c | 2 +- pcu/pcu_coll.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index afc705504..89fad1858 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -751,7 +751,7 @@ void PCU_Switch_Comm(MPI_Comm new_comm) MPI_Comm_compare(new_comm, global_mpi->original_comm, &result); if (result != MPI_IDENT) { pcu_mpi_finalize(&global_mpi); - pcu_mpi_init(new_comm); + global_mpi = pcu_mpi_init(new_comm); } } diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 39405aa6b..225113b13 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -223,7 +223,7 @@ bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) then odd multiples of 2 into even ones, etc... until rank 0 has all inputs merged */ -static int reduce_begin_bit(pcu_mpi_t*) +static int reduce_begin_bit(pcu_mpi_t* mpi) { return 1; } @@ -250,7 +250,7 @@ static int reduce_action(pcu_mpi_t* mpi, int bit) return pcu_coll_recv; } -static int reduce_shift(pcu_mpi_t*, int bit) +static int reduce_shift(pcu_mpi_t* mpi, int bit) { return bit << 1; } @@ -278,7 +278,7 @@ static int bcast_begin_bit(pcu_mpi_t* mpi) return bit; } -static bool bcast_end_bit(pcu_mpi_t *, int bit) +static bool bcast_end_bit(pcu_mpi_t * mpi, int bit) { return bit == 0; } @@ -297,7 +297,7 @@ static int bcast_action(pcu_mpi_t* mpi, int bit) return pcu_coll_send; } -static int bcast_shift(pcu_mpi_t *, int bit) +static int bcast_shift(pcu_mpi_t * mpi, int bit) { return bit >> 1; } @@ -318,7 +318,7 @@ static pcu_pattern bcast = "Parallel Prefix (Scan) Algorithms for MPI". */ -static int scan_up_begin_bit(pcu_mpi_t*) +static int scan_up_begin_bit(pcu_mpi_t* mpi) { return 1; } @@ -371,7 +371,7 @@ static int scan_up_peer(pcu_mpi_t* mpi, int bit) return -1; } -static int scan_up_shift(pcu_mpi_t*, int bit) +static int scan_up_shift(pcu_mpi_t* mpi, int bit) { return bit << 1; } @@ -390,7 +390,7 @@ static int scan_down_begin_bit(pcu_mpi_t* mpi) return 1 << floor_log2(pcu_mpi_size(mpi)); } -static bool scan_down_end_bit(pcu_mpi_t*, int bit) +static bool scan_down_end_bit(pcu_mpi_t* mpi, int bit) { return bit == 1; } @@ -440,7 +440,7 @@ static int scan_down_peer(pcu_mpi_t * mpi, int bit) return -1; } -static int scan_down_shift(pcu_mpi_t*, int bit) +static int scan_down_shift(pcu_mpi_t* mpi, int bit) { return bit >> 1; } From a5be7ae6843c2234b182b7d64afbff4ee024af3f Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 02:05:18 -0400 Subject: [PATCH 018/141] remove virtual function table for pcu_mpi Throughout the code base it was assumed that the mpi functions were implemented through pmpi and so no other implementation really could have been implemented. This change just removes the complexity from having the vtable. --- pcu/pcu_mpi.h | 16 +--------------- pcu/pcu_pmpi.c | 13 ++----------- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 3c2dcc1f9..1bed6d6c7 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -23,28 +23,14 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); -struct pcu_mpi_t_t; typedef struct { - int (*size)(struct pcu_mpi_t_t*); - int (*rank)(struct pcu_mpi_t_t*); - void (*send)(struct pcu_mpi_t_t *, pcu_message* m, MPI_Comm comm); - bool (*done)(struct pcu_mpi_t_t*, pcu_message* m); - bool (*receive)(struct pcu_mpi_t_t*, pcu_message* m, MPI_Comm comm); - void (*construct)(struct pcu_mpi_t_t*, MPI_Comm comm); - void (*destruct)(struct pcu_mpi_t_t*); -} pcu_mpi_vtable; - -struct pcu_mpi_t_t -{ - pcu_mpi_vtable const* vtable; MPI_Comm original_comm; MPI_Comm user_comm; MPI_Comm coll_comm; int rank; int size; -}; -typedef struct pcu_mpi_t_t pcu_mpi_t; +} pcu_mpi_t; int pcu_mpi_size(pcu_mpi_t*); int pcu_mpi_rank(pcu_mpi_t*); diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 44b53c762..e8f546d70 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -18,14 +18,6 @@ bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); void pcu_pmpi_destruct(pcu_mpi_t* self); void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm); -static pcu_mpi_vtable pcu_pmpi_vtable = -{ .size = pcu_pmpi_size, - .rank = pcu_pmpi_rank, - .send = pcu_pmpi_send, - .done = pcu_pmpi_done, - .receive = pcu_pmpi_receive, - .construct = pcu_pmpi_construct, - .destruct = pcu_pmpi_destruct }; pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) { @@ -36,7 +28,7 @@ pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) void pcu_pmpi_finalize(pcu_mpi_t** m) { - (*m)->vtable->destruct(*m); + pcu_pmpi_destruct(*m); free(*m); *m = NULL; } @@ -50,7 +42,6 @@ void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { MPI_Comm_dup(comm,&(self->coll_comm)); MPI_Comm_size(comm,&(self->size)); MPI_Comm_rank(comm,&(self->rank)); - self->vtable = &pcu_pmpi_vtable; } int pcu_pmpi_size(pcu_mpi_t* self) @@ -116,4 +107,4 @@ bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) comm, MPI_STATUS_IGNORE); return true; -} \ No newline at end of file +} From 064ea1348cc3e732f68531584ff2761dcfd7edf6 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 02:52:17 -0400 Subject: [PATCH 019/141] Put PCU state data into a structure --- pcu/pcu.c | 178 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 91 insertions(+), 87 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index 89fad1858..8dd48f0a2 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -42,14 +42,18 @@ #include /*INT_MAX*/ #include /*abort*/ -enum state { uninit, init }; -static enum state global_state = uninit; -static pcu_msg global_pmsg; -static pcu_mpi_t* global_mpi; +typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; +typedef struct { + pcu_state_enum state; + pcu_msg pmsg; + pcu_mpi_t* mpi; +} pcu_t; + +static pcu_t global_pcu = {.state=pcu_state_uninit}; static pcu_msg* get_msg() { - return &global_pmsg; + return &(global_pcu.pmsg); } /** \brief Initializes the PCU library. @@ -59,11 +63,11 @@ static pcu_msg* get_msg() */ int PCU_Comm_Init(void) { - if (global_state != uninit) + if (global_pcu.state != pcu_state_uninit) reel_fail("nested calls to Comm_Init"); - global_mpi = pcu_mpi_init(MPI_COMM_WORLD); - pcu_make_msg(&global_pmsg); - global_state = init; + global_pcu.mpi = pcu_mpi_init(MPI_COMM_WORLD); + pcu_make_msg(&(global_pcu.pmsg)); + global_pcu.state = pcu_state_init; /* turn ordering on by default, call PCU_Comm_Order(false) after PCU_Comm_Init to disable this */ @@ -77,13 +81,13 @@ int PCU_Comm_Init(void) */ int PCU_Comm_Free(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Free called before Comm_Init"); - if (global_pmsg.order) - pcu_order_free(global_pmsg.order); - pcu_free_msg(&global_pmsg); - pcu_mpi_finalize(&global_mpi); - global_state = uninit; + if (global_pcu.pmsg.order) + pcu_order_free(global_pcu.pmsg.order); + pcu_free_msg(&global_pcu.pmsg); + pcu_mpi_finalize(&global_pcu.mpi); + global_pcu.state = pcu_state_uninit; return PCU_SUCCESS; } @@ -98,9 +102,9 @@ int PCU_Comm_Free(void) */ int PCU_Comm_Self(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(global_mpi); + return pcu_mpi_rank(global_pcu.mpi); } /** \brief Returns the number of threads in the program. @@ -109,9 +113,9 @@ int PCU_Comm_Self(void) */ int PCU_Comm_Peers(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(global_mpi); + return pcu_mpi_size(global_pcu.mpi); } /** \brief Begins a PCU communication phase. @@ -122,9 +126,9 @@ int PCU_Comm_Peers(void) */ void PCU_Comm_Begin(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(global_mpi, get_msg()); + pcu_msg_start(global_pcu.mpi, get_msg()); } /** \brief Packs data to be sent to \a to_rank. @@ -135,9 +139,9 @@ void PCU_Comm_Begin(void) */ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) reel_fail("Invalid rank in Comm_Pack"); if ( size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); @@ -156,9 +160,9 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) */ int PCU_Comm_Send(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(global_mpi, get_msg()); + pcu_msg_send(global_pcu.mpi, get_msg()); return PCU_SUCCESS; } @@ -175,12 +179,12 @@ int PCU_Comm_Send(void) */ bool PCU_Comm_Listen(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Listen called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) - return pcu_order_receive(global_mpi, m->order, m); - return pcu_msg_receive(global_mpi, m); + return pcu_order_receive(global_pcu.mpi, m->order, m); + return pcu_msg_receive(global_pcu.mpi, m); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -188,7 +192,7 @@ bool PCU_Comm_Listen(void) */ int PCU_Comm_Sender(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Sender called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -201,7 +205,7 @@ int PCU_Comm_Sender(void) */ bool PCU_Comm_Unpacked(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Unpacked called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -221,7 +225,7 @@ bool PCU_Comm_Unpacked(void) */ int PCU_Comm_Unpack(void* data, size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Unpack called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -233,7 +237,7 @@ int PCU_Comm_Unpack(void* data, size_t size) void PCU_Comm_Order(bool on) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Order called before Comm_Init"); pcu_msg* m = get_msg(); if (on && (!m->order)) @@ -247,9 +251,9 @@ void PCU_Comm_Order(bool on) /** \brief Blocking barrier over all threads. */ void PCU_Barrier(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Barrier called before Comm_Init"); - pcu_barrier(global_mpi, &(get_msg()->coll)); + pcu_barrier(global_pcu.mpi, &(get_msg()->coll)); } /** \brief Performs an Allreduce sum of double arrays. @@ -260,9 +264,9 @@ void PCU_Barrier(void) */ void PCU_Add_Doubles(double* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); } double PCU_Add_Double(double x) @@ -277,9 +281,9 @@ double PCU_Add_Double(double x) */ void PCU_Min_Doubles(double* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); } double PCU_Min_Double(double x) @@ -294,9 +298,9 @@ double PCU_Min_Double(double x) */ void PCU_Max_Doubles(double* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); } double PCU_Max_Double(double x) @@ -311,9 +315,9 @@ double PCU_Max_Double(double x) */ void PCU_Add_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); } int PCU_Add_Int(int x) @@ -328,9 +332,9 @@ int PCU_Add_Int(int x) */ void PCU_Add_Longs(long* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); } long PCU_Add_Long(long x) @@ -345,9 +349,9 @@ long PCU_Add_Long(long x) */ void PCU_Add_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); } size_t PCU_Add_SizeT(size_t x) @@ -361,9 +365,9 @@ size_t PCU_Add_SizeT(size_t x) /** \brief Performs an Allreduce minimum of size_t unsigned integers */ void PCU_Min_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); } size_t PCU_Min_SizeT(size_t x) { @@ -376,9 +380,9 @@ size_t PCU_Min_SizeT(size_t x) { /** \brief Performs an Allreduce maximum of size_t unsigned integers */ void PCU_Max_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); } size_t PCU_Max_SizeT(size_t x) { @@ -396,13 +400,13 @@ size_t PCU_Max_SizeT(size_t x) { */ void PCU_Exscan_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Exscan_Ints called before Comm_Init"); int* originals; NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -420,13 +424,13 @@ int PCU_Exscan_Int(int x) /** \brief See PCU_Exscan_Ints */ void PCU_Exscan_Longs(long* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Exscan_Longs called before Comm_Init"); long* originals; NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -445,9 +449,9 @@ long PCU_Exscan_Long(long x) */ void PCU_Min_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); } int PCU_Min_Int(int x) @@ -462,9 +466,9 @@ int PCU_Min_Int(int x) */ void PCU_Max_Ints(int* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); } int PCU_Max_Int(int x) @@ -478,9 +482,9 @@ int PCU_Max_Int(int x) */ void PCU_Max_Longs(long* p, size_t n) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(global_mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); } long PCU_Max_Long(long x) @@ -509,43 +513,43 @@ int PCU_And(int c) */ int PCU_Proc_Self(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(global_mpi); + return pcu_mpi_rank(global_pcu.mpi); } /** \brief Returns the number of processes. */ int PCU_Proc_Peers(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(global_mpi); + return pcu_mpi_size(global_pcu.mpi); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ int PCU_Comm_Rank(int* rank) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(global_mpi); + *rank = pcu_mpi_rank(global_pcu.mpi); return PCU_SUCCESS; } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ int PCU_Comm_Size(int* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(global_mpi); + *size = pcu_mpi_size(global_pcu.mpi); return PCU_SUCCESS; } /** \brief Returns true iff PCU has been initialized */ bool PCU_Comm_Initialized(void) { - return global_state == init; + return global_pcu.state == pcu_state_init; } /** \brief Deprecated, see PCU_Comm_Begin. @@ -553,9 +557,9 @@ bool PCU_Comm_Initialized(void) int PCU_Comm_Start(PCU_Method method) { (void)method; //warning silencer - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(global_mpi,get_msg()); + pcu_msg_start(global_pcu.mpi,get_msg()); return PCU_SUCCESS; } @@ -566,9 +570,9 @@ int PCU_Comm_Start(PCU_Method method) */ int PCU_Comm_Packed(int to_rank, size_t* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) reel_fail("Invalid rank in Comm_Packed"); *size = pcu_msg_packed(get_msg(),to_rank); return PCU_SUCCESS; @@ -585,9 +589,9 @@ int PCU_Comm_Packed(int to_rank, size_t* size) */ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) reel_fail("Invalid rank in Comm_Write"); pcu_msg* msg = get_msg(); PCU_MSG_PACK(msg,to_rank,size); @@ -618,7 +622,7 @@ bool PCU_Comm_Receive(void) */ bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Read called before Comm_Init"); if (!PCU_Comm_Receive()) return false; @@ -649,7 +653,7 @@ static void append(char* s, size_t size, const char* format, ...) void PCU_Debug_Open(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Debug_Open called before Comm_Init"); const int fanout = 2048; @@ -677,7 +681,7 @@ void PCU_Debug_Open(void) /** \brief like fprintf, contents go to debugN.txt */ void PCU_Debug_Print(const char* format, ...) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Debug_Print called before Comm_Init"); pcu_msg* msg = get_msg(); if ( ! msg->file) @@ -692,7 +696,7 @@ void PCU_Debug_Print(const char* format, ...) /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ int PCU_Comm_From(int* from_rank) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_From called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -709,7 +713,7 @@ int PCU_Comm_From(int* from_rank) */ int PCU_Comm_Received(size_t* size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Received called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -727,7 +731,7 @@ int PCU_Comm_Received(size_t* size) */ void* PCU_Comm_Extract(size_t size) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Extract called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) @@ -745,13 +749,13 @@ void* PCU_Comm_Extract(size_t size) */ void PCU_Switch_Comm(MPI_Comm new_comm) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Switch_Comm called before Comm_Init"); int result; - MPI_Comm_compare(new_comm, global_mpi->original_comm, &result); + MPI_Comm_compare(new_comm, global_pcu.mpi->original_comm, &result); if (result != MPI_IDENT) { - pcu_mpi_finalize(&global_mpi); - global_mpi = pcu_mpi_init(new_comm); + pcu_mpi_finalize(&(global_pcu.mpi)); + global_pcu.mpi = pcu_mpi_init(new_comm); } } @@ -762,9 +766,9 @@ void PCU_Switch_Comm(MPI_Comm new_comm) */ MPI_Comm PCU_Get_Comm(void) { - if (global_state == uninit) + if (global_pcu.state == pcu_state_uninit) reel_fail("Get_Comm called before Comm_Init"); - return global_mpi->original_comm; + return global_pcu.mpi->original_comm; } /** \brief Return the time in seconds since some time in the past From e727d5439582e070f11cb42f4065faaf4593e17b Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 03:07:50 -0400 Subject: [PATCH 020/141] Use consistent initialization functions for MPI. This commit allows the pcu_mpi to be initialized into a stack variable rather than allways allocating on the heap. --- pcu/pcu.c | 66 +++++++++++++++++++++++++------------------------- pcu/pcu_mpi.c | 6 ++--- pcu/pcu_mpi.h | 4 +-- pcu/pcu_pmpi.c | 27 ++++++--------------- pcu/pcu_pmpi.h | 4 +-- 5 files changed, 47 insertions(+), 60 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index 8dd48f0a2..a7f2c26fd 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -46,7 +46,7 @@ typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; typedef struct { pcu_state_enum state; pcu_msg pmsg; - pcu_mpi_t* mpi; + pcu_mpi_t mpi; } pcu_t; static pcu_t global_pcu = {.state=pcu_state_uninit}; @@ -65,7 +65,7 @@ int PCU_Comm_Init(void) { if (global_pcu.state != pcu_state_uninit) reel_fail("nested calls to Comm_Init"); - global_pcu.mpi = pcu_mpi_init(MPI_COMM_WORLD); + pcu_mpi_init(MPI_COMM_WORLD,&global_pcu.mpi); pcu_make_msg(&(global_pcu.pmsg)); global_pcu.state = pcu_state_init; /* turn ordering on by default, call @@ -104,7 +104,7 @@ int PCU_Comm_Self(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(global_pcu.mpi); + return pcu_mpi_rank(&global_pcu.mpi); } /** \brief Returns the number of threads in the program. @@ -115,7 +115,7 @@ int PCU_Comm_Peers(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(global_pcu.mpi); + return pcu_mpi_size(&global_pcu.mpi); } /** \brief Begins a PCU communication phase. @@ -128,7 +128,7 @@ void PCU_Comm_Begin(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(global_pcu.mpi, get_msg()); + pcu_msg_start(&global_pcu.mpi, get_msg()); } /** \brief Packs data to be sent to \a to_rank. @@ -141,7 +141,7 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) reel_fail("Invalid rank in Comm_Pack"); if ( size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); @@ -162,7 +162,7 @@ int PCU_Comm_Send(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(global_pcu.mpi, get_msg()); + pcu_msg_send(&global_pcu.mpi, get_msg()); return PCU_SUCCESS; } @@ -183,8 +183,8 @@ bool PCU_Comm_Listen(void) reel_fail("Comm_Listen called before Comm_Init"); pcu_msg* m = get_msg(); if (m->order) - return pcu_order_receive(global_pcu.mpi, m->order, m); - return pcu_msg_receive(global_pcu.mpi, m); + return pcu_order_receive(&global_pcu.mpi, m->order, m); + return pcu_msg_receive(&global_pcu.mpi, m); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -253,7 +253,7 @@ void PCU_Barrier(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Barrier called before Comm_Init"); - pcu_barrier(global_pcu.mpi, &(get_msg()->coll)); + pcu_barrier(&global_pcu.mpi, &(get_msg()->coll)); } /** \brief Performs an Allreduce sum of double arrays. @@ -266,7 +266,7 @@ void PCU_Add_Doubles(double* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); } double PCU_Add_Double(double x) @@ -283,7 +283,7 @@ void PCU_Min_Doubles(double* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); } double PCU_Min_Double(double x) @@ -300,7 +300,7 @@ void PCU_Max_Doubles(double* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); } double PCU_Max_Double(double x) @@ -317,7 +317,7 @@ void PCU_Add_Ints(int* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); } int PCU_Add_Int(int x) @@ -334,7 +334,7 @@ void PCU_Add_Longs(long* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); } long PCU_Add_Long(long x) @@ -351,7 +351,7 @@ void PCU_Add_SizeTs(size_t* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); } size_t PCU_Add_SizeT(size_t x) @@ -367,7 +367,7 @@ size_t PCU_Add_SizeT(size_t x) void PCU_Min_SizeTs(size_t* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); } size_t PCU_Min_SizeT(size_t x) { @@ -382,7 +382,7 @@ size_t PCU_Min_SizeT(size_t x) { void PCU_Max_SizeTs(size_t* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); } size_t PCU_Max_SizeT(size_t x) { @@ -406,7 +406,7 @@ void PCU_Exscan_Ints(int* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -430,7 +430,7 @@ void PCU_Exscan_Longs(long* p, size_t n) NOTO_MALLOC(originals,n); for (size_t i=0; i < n; ++i) originals[i] = p[i]; - pcu_scan(global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); //convert inclusive scan to exclusive for (size_t i=0; i < n; ++i) p[i] -= originals[i]; @@ -451,7 +451,7 @@ void PCU_Min_Ints(int* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); } int PCU_Min_Int(int x) @@ -468,7 +468,7 @@ void PCU_Max_Ints(int* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); } int PCU_Max_Int(int x) @@ -484,7 +484,7 @@ void PCU_Max_Longs(long* p, size_t n) { if (global_pcu.state == pcu_state_uninit) reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); } long PCU_Max_Long(long x) @@ -515,7 +515,7 @@ int PCU_Proc_Self(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(global_pcu.mpi); + return pcu_mpi_rank(&global_pcu.mpi); } /** \brief Returns the number of processes. @@ -524,7 +524,7 @@ int PCU_Proc_Peers(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(global_pcu.mpi); + return pcu_mpi_size(&global_pcu.mpi); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. @@ -533,7 +533,7 @@ int PCU_Comm_Rank(int* rank) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(global_pcu.mpi); + *rank = pcu_mpi_rank(&global_pcu.mpi); return PCU_SUCCESS; } @@ -542,7 +542,7 @@ int PCU_Comm_Size(int* size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(global_pcu.mpi); + *size = pcu_mpi_size(&global_pcu.mpi); return PCU_SUCCESS; } @@ -559,7 +559,7 @@ int PCU_Comm_Start(PCU_Method method) (void)method; //warning silencer if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(global_pcu.mpi,get_msg()); + pcu_msg_start(&global_pcu.mpi,get_msg()); return PCU_SUCCESS; } @@ -572,7 +572,7 @@ int PCU_Comm_Packed(int to_rank, size_t* size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) reel_fail("Invalid rank in Comm_Packed"); *size = pcu_msg_packed(get_msg(),to_rank); return PCU_SUCCESS; @@ -591,7 +591,7 @@ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { if (global_pcu.state == pcu_state_uninit) reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(global_pcu.mpi))) + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) reel_fail("Invalid rank in Comm_Write"); pcu_msg* msg = get_msg(); PCU_MSG_PACK(msg,to_rank,size); @@ -752,10 +752,10 @@ void PCU_Switch_Comm(MPI_Comm new_comm) if (global_pcu.state == pcu_state_uninit) reel_fail("Switch_Comm called before Comm_Init"); int result; - MPI_Comm_compare(new_comm, global_pcu.mpi->original_comm, &result); + MPI_Comm_compare(new_comm, global_pcu.mpi.original_comm, &result); if (result != MPI_IDENT) { pcu_mpi_finalize(&(global_pcu.mpi)); - global_pcu.mpi = pcu_mpi_init(new_comm); + pcu_mpi_init(new_comm,&global_pcu.mpi); } } @@ -768,7 +768,7 @@ MPI_Comm PCU_Get_Comm(void) { if (global_pcu.state == pcu_state_uninit) reel_fail("Get_Comm called before Comm_Init"); - return global_pcu.mpi->original_comm; + return global_pcu.mpi.original_comm; } /** \brief Return the time in seconds since some time in the past diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index c5dea8baa..40236ef95 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -61,9 +61,9 @@ bool pcu_mpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) check_rank(self, m->peer); return pcu_pmpi_receive(self, m, comm); } -pcu_mpi_t* pcu_mpi_init(MPI_Comm comm) { - return pcu_pmpi_init(comm); +void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi) { + pcu_pmpi_init(comm, mpi); } -void pcu_mpi_finalize(pcu_mpi_t** mpi) { +void pcu_mpi_finalize(pcu_mpi_t* mpi) { pcu_pmpi_finalize(mpi); } diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 1bed6d6c7..9c0fad4f4 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -37,7 +37,7 @@ int pcu_mpi_rank(pcu_mpi_t*); void pcu_mpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); bool pcu_mpi_done(pcu_mpi_t*, pcu_message* m); bool pcu_mpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -pcu_mpi_t* pcu_mpi_init(MPI_Comm comm); -void pcu_mpi_finalize(pcu_mpi_t**); +void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); +void pcu_mpi_finalize(pcu_mpi_t* mpi); #endif diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index e8f546d70..a4280dace 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -15,28 +15,9 @@ void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); -void pcu_pmpi_destruct(pcu_mpi_t* self); -void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm); - -pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm) -{ - pcu_mpi_t* m = (pcu_mpi_t*)malloc(sizeof(pcu_mpi_t)); - pcu_pmpi_construct(m,comm); - return m; -} - -void pcu_pmpi_finalize(pcu_mpi_t** m) +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* self) { - pcu_pmpi_destruct(*m); - free(*m); - *m = NULL; -} -void pcu_pmpi_destruct(pcu_mpi_t* self) { - MPI_Comm_free(&(self->user_comm)); - MPI_Comm_free(&(self->coll_comm)); -} -void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { self->original_comm = comm; MPI_Comm_dup(comm,&(self->user_comm)); MPI_Comm_dup(comm,&(self->coll_comm)); @@ -44,6 +25,12 @@ void pcu_pmpi_construct(pcu_mpi_t* self, MPI_Comm comm) { MPI_Comm_rank(comm,&(self->rank)); } +void pcu_pmpi_finalize(pcu_mpi_t* self) +{ + MPI_Comm_free(&(self->user_comm)); + MPI_Comm_free(&(self->coll_comm)); +} + int pcu_pmpi_size(pcu_mpi_t* self) { return self->size; diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index cdfbbf699..36507421d 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -14,8 +14,8 @@ #include -pcu_mpi_t* pcu_pmpi_init(MPI_Comm comm); -void pcu_pmpi_finalize(pcu_mpi_t** m); +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* mpi); +void pcu_pmpi_finalize(pcu_mpi_t* m); int pcu_pmpi_size(pcu_mpi_t* self); int pcu_pmpi_rank(pcu_mpi_t* self); void pcu_pmpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); From 47bf6d57fa154f55087fbf9fbbba43faae9d0ba5 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 14:08:36 -0400 Subject: [PATCH 021/141] Add PCU2 interface that takes the PCU state struct --- pcu/CMakeLists.txt | 1 + pcu/PCU2.h | 137 ++++++++ pcu/pcu.c | 392 ++++------------------- pcu/pcu2.c | 783 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 985 insertions(+), 328 deletions(-) create mode 100644 pcu/PCU2.h create mode 100644 pcu/pcu2.c diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index bdd99e6f4..8c5598168 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -10,6 +10,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES pcu.c + pcu2.c pcu_aa.c pcu_coll.c pcu_io.c diff --git a/pcu/PCU2.h b/pcu/PCU2.h new file mode 100644 index 000000000..3601ff029 --- /dev/null +++ b/pcu/PCU2.h @@ -0,0 +1,137 @@ +/****************************************************************************** + + Copyright 2011 Scientific Computation Research Center, + Rensselaer Polytechnic Institute. All rights reserved. + + This work is open source software, licensed under the terms of the + BSD license as described in the LICENSE file in the top-level directory. + +*******************************************************************************/ +#ifndef PCU2_H +#define PCU2_H + +#define PCU_SUCCESS 0 +#define PCU_FAILURE -1 + +#include + +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif + +// FOR pcu_state_type ... use opaque ptr instead? +#include "pcu_msg.h" + +typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; +typedef struct { + pcu_state_enum state; + pcu_msg pmsg; + pcu_mpi_t mpi; +} pcu_t; + +/*library init/finalize*/ +int PCU_Comm_Init_2(pcu_t* pcu); +int PCU_Comm_Free_2(pcu_t* pcu); + +/*rank/size functions*/ +int PCU_Comm_Self_2(pcu_t* pcu); +int PCU_Comm_Peers_2(pcu_t* pcu); + +/*recommended message passing API*/ +void PCU_Comm_Begin_2(pcu_t* pcu); +int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size); +int PCU_Comm_Send_2(pcu_t* pcu); +bool PCU_Comm_Receive_2(pcu_t* pcu); +bool PCU_Comm_Listen_2(pcu_t* pcu); +int PCU_Comm_Sender_2(pcu_t* pcu); +bool PCU_Comm_Unpacked_2(pcu_t* pcu); +int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size); + +/*turns deterministic ordering for the + above API on/off*/ +void PCU_Comm_Order_2(pcu_t* pcu, bool on); + +/*collective operations*/ +void PCU_Barrier_2(pcu_t* pcu); +void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n); +double PCU_Add_Double_2(pcu_t* pcu, double x); +void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n); +double PCU_Min_Double_2(pcu_t* pcu, double x); +void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n); +double PCU_Max_Double_2(pcu_t* pcu, double x); +void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Add_Int_2(pcu_t* pcu, int x); +void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n); +long PCU_Add_Long_2(pcu_t* pcu, long x); +void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Exscan_Int_2(pcu_t* pcu, int x); +void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n); +long PCU_Exscan_Long_2(pcu_t* pcu, long x); +void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); +size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x); +void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); +size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x); +void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); +size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x); +void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Min_Int_2(pcu_t* pcu, int x); +void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n); +int PCU_Max_Int_2(pcu_t* pcu, int x); +void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n); +long PCU_Max_Long_2(pcu_t* pcu, long x); +int PCU_Or_2(pcu_t* pcu, int c); +int PCU_And_2(pcu_t* pcu, int c); + +/*process-level self/peers (mpi wrappers)*/ +int PCU_Proc_Self_2(pcu_t* pcu); +int PCU_Proc_Peers_2(pcu_t* pcu); + +/*IPComMan replacement API*/ +int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size); +bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size); + +/*Debug file I/O API*/ +void PCU_Debug_Open_2(pcu_t* pcu); +void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list args); +#ifdef __GNUC__ +void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) + __attribute__((format(printf,2,3))); +#else +void PCU_Debug_Print(pcu_t* pcu, const char* format, ...); +#endif + +/*lesser-used APIs*/ +bool PCU_Comm_Initialized_2(pcu_t* pcu); +int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size); +int PCU_Comm_From_2(pcu_t* pcu, int* from_rank); +int PCU_Comm_Received_2(pcu_t* pcu, size_t* size); +void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size); +int PCU_Comm_Rank_2(pcu_t* pcu, int* rank); +int PCU_Comm_Size_2(pcu_t* pcu, int* size); + +int PCU_Comm_Start_2(pcu_t* pcu); + +/*special MPI_Comm replacement API*/ +void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm); +MPI_Comm PCU_Get_Comm_2(pcu_t* pcu); + +/*stack trace helpers using GNU/Linux*/ +void PCU_Protect_2(pcu_t* pcu); + +/*MPI_Wtime_() equivalent*/ +double PCU_Time_2(pcu_t* pcu); + +/*Memory usage*/ +double PCU_GetMem_2(pcu_t* pcu); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/pcu/pcu.c b/pcu/pcu.c index a7f2c26fd..d4679fc86 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -32,6 +32,7 @@ #include #include #include "PCU.h" +#include "PCU2.h" #include "pcu_msg.h" #include "pcu_order.h" #include "noto_malloc.h" @@ -42,20 +43,9 @@ #include /*INT_MAX*/ #include /*abort*/ -typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; -typedef struct { - pcu_state_enum state; - pcu_msg pmsg; - pcu_mpi_t mpi; -} pcu_t; static pcu_t global_pcu = {.state=pcu_state_uninit}; -static pcu_msg* get_msg() -{ - return &(global_pcu.pmsg); -} - /** \brief Initializes the PCU library. \details This function must be called by all MPI processes before calling any other PCU functions. @@ -63,16 +53,7 @@ static pcu_msg* get_msg() */ int PCU_Comm_Init(void) { - if (global_pcu.state != pcu_state_uninit) - reel_fail("nested calls to Comm_Init"); - pcu_mpi_init(MPI_COMM_WORLD,&global_pcu.mpi); - pcu_make_msg(&(global_pcu.pmsg)); - global_pcu.state = pcu_state_init; - /* turn ordering on by default, call - PCU_Comm_Order(false) after PCU_Comm_Init - to disable this */ - PCU_Comm_Order(true); - return PCU_SUCCESS; + PCU_Comm_Init_2(&global_pcu); } /** \brief Frees all PCU library structures. @@ -81,14 +62,7 @@ int PCU_Comm_Init(void) */ int PCU_Comm_Free(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Free called before Comm_Init"); - if (global_pcu.pmsg.order) - pcu_order_free(global_pcu.pmsg.order); - pcu_free_msg(&global_pcu.pmsg); - pcu_mpi_finalize(&global_pcu.mpi); - global_pcu.state = pcu_state_uninit; - return PCU_SUCCESS; + return PCU_Comm_Free_2(&global_pcu); } /** \brief Returns the communication rank of the calling thread. @@ -102,9 +76,7 @@ int PCU_Comm_Free(void) */ int PCU_Comm_Self(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(&global_pcu.mpi); + return PCU_Comm_Self_2(&global_pcu); } /** \brief Returns the number of threads in the program. @@ -113,9 +85,7 @@ int PCU_Comm_Self(void) */ int PCU_Comm_Peers(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(&global_pcu.mpi); + return PCU_Comm_Peers_2(&global_pcu); } /** \brief Begins a PCU communication phase. @@ -126,9 +96,7 @@ int PCU_Comm_Peers(void) */ void PCU_Comm_Begin(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(&global_pcu.mpi, get_msg()); + PCU_Comm_Begin_2(&global_pcu); } /** \brief Packs data to be sent to \a to_rank. @@ -139,16 +107,7 @@ void PCU_Comm_Begin(void) */ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) - reel_fail("Invalid rank in Comm_Pack"); - if ( size > (size_t)INT_MAX ) { - fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); - abort(); - } - memcpy(pcu_msg_pack(get_msg(),to_rank,size),data,size); - return PCU_SUCCESS; + return PCU_Comm_Pack_2(&global_pcu, to_rank, data, size); } /** \brief Sends all buffers for this communication phase. @@ -160,10 +119,7 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) */ int PCU_Comm_Send(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(&global_pcu.mpi, get_msg()); - return PCU_SUCCESS; + return PCU_Comm_Send_2(&global_pcu); } /** \brief Tries to receive a buffer for this communication phase. @@ -179,12 +135,7 @@ int PCU_Comm_Send(void) */ bool PCU_Comm_Listen(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Listen called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_receive(&global_pcu.mpi, m->order, m); - return pcu_msg_receive(&global_pcu.mpi, m); + return PCU_Comm_Listen_2(&global_pcu); } /** \brief Returns in * \a from_rank the sender of the current received buffer. @@ -192,12 +143,7 @@ bool PCU_Comm_Listen(void) */ int PCU_Comm_Sender(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Sender called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_received_from(m->order); - return pcu_msg_received_from(m); + return PCU_Comm_Sender_2(&global_pcu); } /** \brief Returns true if the current received buffer has been unpacked. @@ -205,12 +151,7 @@ int PCU_Comm_Sender(void) */ bool PCU_Comm_Unpacked(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Unpacked called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_unpacked(m->order); - return pcu_msg_unpacked(m); + return PCU_Comm_Unpacked_2(&global_pcu); } /** \brief Unpacks a block of data from the current received buffer. @@ -225,35 +166,18 @@ bool PCU_Comm_Unpacked(void) */ int PCU_Comm_Unpack(void* data, size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Unpack called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - memcpy(data,pcu_order_unpack(m->order,size),size); - else - memcpy(data,pcu_msg_unpack(m,size),size); - return PCU_SUCCESS; + return PCU_Comm_Unpack_2(&global_pcu, data, size); } void PCU_Comm_Order(bool on) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Order called before Comm_Init"); - pcu_msg* m = get_msg(); - if (on && (!m->order)) - m->order = pcu_order_new(); - if ((!on) && m->order) { - pcu_order_free(m->order); - m->order = NULL; - } + PCU_Comm_Order_2(&global_pcu, on); } /** \brief Blocking barrier over all threads. */ void PCU_Barrier(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&global_pcu.mpi, &(get_msg()->coll)); + PCU_Barrier_2(&global_pcu); } /** \brief Performs an Allreduce sum of double arrays. @@ -264,132 +188,92 @@ void PCU_Barrier(void) */ void PCU_Add_Doubles(double* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); + PCU_Add_Doubles_2(&global_pcu, p, n); } double PCU_Add_Double(double x) { - double a[1]; - a[0] = x; - PCU_Add_Doubles(a, 1); - return a[0]; + return PCU_Add_Double_2(&global_pcu, x); } /** \brief Performs an Allreduce minimum of double arrays. */ void PCU_Min_Doubles(double* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); + PCU_Min_Doubles_2(&global_pcu, p, n); } double PCU_Min_Double(double x) { - double a[1]; - a[0] = x; - PCU_Min_Doubles(a, 1); - return a[0]; + return PCU_Min_Double_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of double arrays. */ void PCU_Max_Doubles(double* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); + PCU_Max_Doubles_2(&global_pcu, p, n); } double PCU_Max_Double(double x) { - double a[1]; - a[0] = x; - PCU_Max_Doubles(a, 1); - return a[0]; + return PCU_Max_Double_2(&global_pcu, x); } /** \brief Performs an Allreduce sum of integers */ void PCU_Add_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); + PCU_Add_Ints_2(&global_pcu, p, n); } int PCU_Add_Int(int x) { - int a[1]; - a[0] = x; - PCU_Add_Ints(a, 1); - return a[0]; + return PCU_Add_Int_2(&global_pcu, x); } /** \brief Performs an Allreduce sum of long integers */ void PCU_Add_Longs(long* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); + PCU_Add_Longs_2(&global_pcu, p, n); } long PCU_Add_Long(long x) { - long a[1]; - a[0] = x; - PCU_Add_Longs(a, 1); - return a[0]; + return PCU_Add_Long_2(&global_pcu, x); } /** \brief Performs an Allreduce sum of size_t unsigned integers */ void PCU_Add_SizeTs(size_t* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); + PCU_Add_SizeTs_2(&global_pcu, p, n); } size_t PCU_Add_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Add_SizeTs(a, 1); - return a[0]; + return PCU_Add_SizeT_2(&global_pcu, x); } /** \brief Performs an Allreduce minimum of size_t unsigned integers */ void PCU_Min_SizeTs(size_t* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); + PCU_Min_SizeTs_2(&global_pcu, p, n); } size_t PCU_Min_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Min_SizeTs(a, 1); - return a[0]; + return PCU_Min_SizeT_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of size_t unsigned integers */ void PCU_Max_SizeTs(size_t* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); + PCU_Max_SizeTs_2(&global_pcu, p, n); } size_t PCU_Max_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Max_SizeTs(a, 1); - return a[0]; + return PCU_Max_SizeT_2(&global_pcu, x); } /** \brief Performs an exclusive prefix sum of integer arrays. @@ -400,156 +284,105 @@ size_t PCU_Max_SizeT(size_t x) { */ void PCU_Exscan_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Exscan_Ints called before Comm_Init"); - int* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); + PCU_Exscan_Ints_2(&global_pcu, p, n); } int PCU_Exscan_Int(int x) { - int a[1]; - a[0] = x; - PCU_Exscan_Ints(a, 1); - return a[0]; + return PCU_Exscan_Int_2(&global_pcu, x); } /** \brief See PCU_Exscan_Ints */ void PCU_Exscan_Longs(long* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Exscan_Longs called before Comm_Init"); - long* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&global_pcu.mpi, &(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); + PCU_Exscan_Longs_2(&global_pcu, p, n); } long PCU_Exscan_Long(long x) { - long a[1]; - a[0] = x; - PCU_Exscan_Longs(a, 1); - return a[0]; + return PCU_Exscan_Long_2(&global_pcu, x); } /** \brief Performs an Allreduce minimum of int arrays. */ void PCU_Min_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); + PCU_Min_Ints_2(&global_pcu, p, n); } int PCU_Min_Int(int x) { - int a[1]; - a[0] = x; - PCU_Min_Ints(a, 1); - return a[0]; + return PCU_Min_Int_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of int arrays. */ void PCU_Max_Ints(int* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); + PCU_Max_Ints_2(&global_pcu, p, n); } int PCU_Max_Int(int x) { - int a[1]; - a[0] = x; - PCU_Max_Ints(a, 1); - return a[0]; + return PCU_Max_Int_2(&global_pcu, x); } /** \brief Performs an Allreduce maximum of long arrays. */ void PCU_Max_Longs(long* p, size_t n) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&global_pcu.mpi, &(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); + PCU_Max_Longs_2(&global_pcu, p, n); } long PCU_Max_Long(long x) { - long a[1]; - a[0] = x; - PCU_Max_Longs(a, 1); - return a[0]; + return PCU_Max_Long_2(&global_pcu, x); } /** \brief Performs a parallel logical OR reduction */ int PCU_Or(int c) { - return PCU_Max_Int(c); + return PCU_Or_2(&global_pcu, c); } /** \brief Performs a parallel logical AND reduction */ int PCU_And(int c) { - return PCU_Min_Int(c); + return PCU_And_2(&global_pcu, c); } /** \brief Returns the unique rank of the calling process. */ int PCU_Proc_Self(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(&global_pcu.mpi); + return PCU_Proc_Self_2(&global_pcu); } /** \brief Returns the number of processes. */ int PCU_Proc_Peers(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(&global_pcu.mpi); + return PCU_Proc_Peers_2(&global_pcu); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ int PCU_Comm_Rank(int* rank) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(&global_pcu.mpi); - return PCU_SUCCESS; + return PCU_Comm_Rank_2(&global_pcu, rank); } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ int PCU_Comm_Size(int* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(&global_pcu.mpi); - return PCU_SUCCESS; + return PCU_Comm_Size_2(&global_pcu, size); } /** \brief Returns true iff PCU has been initialized */ bool PCU_Comm_Initialized(void) { - return global_pcu.state == pcu_state_init; + return PCU_Comm_Initialized_2(&global_pcu); } /** \brief Deprecated, see PCU_Comm_Begin. @@ -557,10 +390,7 @@ bool PCU_Comm_Initialized(void) int PCU_Comm_Start(PCU_Method method) { (void)method; //warning silencer - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(&global_pcu.mpi,get_msg()); - return PCU_SUCCESS; + return PCU_Comm_Start_2(&global_pcu); } /** \brief Returns in * \a size the number of bytes being sent to \a to_rank. @@ -570,12 +400,7 @@ int PCU_Comm_Start(PCU_Method method) */ int PCU_Comm_Packed(int to_rank, size_t* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) - reel_fail("Invalid rank in Comm_Packed"); - *size = pcu_msg_packed(get_msg(),to_rank); - return PCU_SUCCESS; + return PCU_Comm_Packed_2(&global_pcu, to_rank, size); } /** \brief Packs a message to be sent to \a to_rank. @@ -589,23 +414,13 @@ int PCU_Comm_Packed(int to_rank, size_t* size) */ int PCU_Comm_Write(int to_rank, const void* data, size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&global_pcu.mpi))) - reel_fail("Invalid rank in Comm_Write"); - pcu_msg* msg = get_msg(); - PCU_MSG_PACK(msg,to_rank,size); - memcpy(pcu_msg_pack(msg,to_rank,size),data,size); - return PCU_SUCCESS; + return PCU_Comm_Write_2(&global_pcu, to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ bool PCU_Comm_Receive(void) { - while (PCU_Comm_Unpacked()) - if (!PCU_Comm_Listen()) - return false; - return true; + return PCU_Comm_Receive_2(&global_pcu); } /** \brief Receives a message for this communication phase. @@ -622,88 +437,30 @@ bool PCU_Comm_Receive(void) */ bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Read called before Comm_Init"); - if (!PCU_Comm_Receive()) - return false; - *from_rank = PCU_Comm_Sender(); - PCU_COMM_UNPACK(*size); - *data = PCU_Comm_Extract(*size); - return true; + return PCU_Comm_Read_2(&global_pcu, from_rank, data, size); } -static void safe_mkdir(const char* path, mode_t mode) -{ - int err; - errno = 0; - err = mkdir(path, mode); - if (err != 0 && errno != EEXIST) - reel_fail("PCU: could not create directory \"%s\"\n", path); -} -static void append(char* s, size_t size, const char* format, ...) -{ - int len = strlen(s); - va_list ap; - va_start(ap, format); - vsnprintf(s + len, size - len, format, ap); - va_end(ap); -} void PCU_Debug_Open(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Debug_Open called before Comm_Init"); - - const int fanout = 2048; - const int bufsize = 1024; - char* path = noto_malloc(bufsize); - path[0] = '\0'; - if (PCU_Comm_Peers() > fanout) { - mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - strcpy(path, "debug/"); - safe_mkdir(path, dir_perm); - int self = PCU_Comm_Self(); - append(path, bufsize, "%d/", self / fanout); - if (self % fanout == 0) - safe_mkdir(path, dir_perm); - PCU_Barrier(); - } - - append(path,bufsize, "%s", "debug"); - pcu_msg* msg = get_msg(); - if ( ! msg->file) - msg->file = pcu_open_parallel(path,"txt"); - noto_free(path); + PCU_Debug_Open_2(&global_pcu); } /** \brief like fprintf, contents go to debugN.txt */ void PCU_Debug_Print(const char* format, ...) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Debug_Print called before Comm_Init"); - pcu_msg* msg = get_msg(); - if ( ! msg->file) - return; //Print is a no-op if no file is open - va_list ap; - va_start(ap,format); - vfprintf(msg->file,format,ap); - va_end(ap); - fflush(msg->file); + va_list arglist; + va_start(arglist, format); + PCU_Debug_Printv_2(&global_pcu, format, arglist); + va_end(arglist); } /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ int PCU_Comm_From(int* from_rank) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_From called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - *from_rank = pcu_order_received_from(m->order); - else - *from_rank = pcu_msg_received_from(m); - return PCU_SUCCESS; + return PCU_Comm_From_2(&global_pcu, from_rank); } /** \brief Returns in * \a size the bytes in the current received buffer @@ -713,14 +470,7 @@ int PCU_Comm_From(int* from_rank) */ int PCU_Comm_Received(size_t* size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Received called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - *size = pcu_order_received_size(m->order); - else - *size = pcu_msg_received_size(m); - return PCU_SUCCESS; + return PCU_Comm_Received_2(&global_pcu, size); } /** \brief Extracts a block of data from the current received buffer. @@ -731,12 +481,7 @@ int PCU_Comm_Received(size_t* size) */ void* PCU_Comm_Extract(size_t size) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Comm_Extract called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_unpack(m->order,size); - return pcu_msg_unpack(m,size); + return PCU_Comm_Extract_2(&global_pcu, size); } /** \brief Reinitializes PCU with a new MPI communicator. @@ -749,14 +494,7 @@ void* PCU_Comm_Extract(size_t size) */ void PCU_Switch_Comm(MPI_Comm new_comm) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Switch_Comm called before Comm_Init"); - int result; - MPI_Comm_compare(new_comm, global_pcu.mpi.original_comm, &result); - if (result != MPI_IDENT) { - pcu_mpi_finalize(&(global_pcu.mpi)); - pcu_mpi_init(new_comm,&global_pcu.mpi); - } + PCU_Switch_Comm_2(&global_pcu, new_comm); } /** \brief Return the current MPI communicator @@ -766,19 +504,17 @@ void PCU_Switch_Comm(MPI_Comm new_comm) */ MPI_Comm PCU_Get_Comm(void) { - if (global_pcu.state == pcu_state_uninit) - reel_fail("Get_Comm called before Comm_Init"); - return global_pcu.mpi.original_comm; + PCU_Get_Comm_2(&global_pcu); } /** \brief Return the time in seconds since some time in the past */ double PCU_Time(void) { - return MPI_Wtime(); + return PCU_Time_2(&global_pcu); } void PCU_Protect(void) { - reel_protect(); + PCU_Protect_2(&global_pcu); } diff --git a/pcu/pcu2.c b/pcu/pcu2.c new file mode 100644 index 000000000..001ac9ed2 --- /dev/null +++ b/pcu/pcu2.c @@ -0,0 +1,783 @@ +/****************************************************************************** + + Copyright 2011 Scientific Computation Research Center, + Rensselaer Polytechnic Institute. All rights reserved. + + This work is open source software, licensed under the terms of the + BSD license as described in the LICENSE file in the top-level directory. + +*******************************************************************************/ +/** \file pcu->c + \brief The PCU communication interface */ +/** \page pcu PCU + PCU (the Parallel Control Utility) is a library for parallel computation + based on MPI. + PCU provides three things to users: + 1. A hybrid phased message passing system + 2. Hybrid collective operations + + Phased message passing is similar to Bulk Synchronous Parallel. + All messages are exchanged in a phase, which is a collective operation + involving all threads in the parallel program. + During a phase, the following events happen in sequence: + 1. All threads send non-blocking messages to other threads + 2. All threads receive all messages sent to them during this phase + PCU provides termination detection, which is the ability to detect when all + messages have been received without prior knowledge of which threads + are sending to which. + + The API documentation is here: pcu->c +*/ + +#include +#include +#include "PCU2.h" +#include "pcu_msg.h" +#include "pcu_order.h" +#include "noto_malloc.h" +#include "reel.h" +#include /*required for mode_t for mkdir on some systems*/ +#include /*using POSIX mkdir call for SMB "foo/" path*/ +#include /* for checking the error from mkdir */ +#include /*INT_MAX*/ +#include /*abort*/ + + +static pcu_msg* get_msg(pcu_t * pcu) +{ + return &(pcu->pmsg); +} + +/** \brief Initializes the PCU library. + \details This function must be called by all MPI processes before + calling any other PCU functions. + MPI_Init or MPI_Init_thread should be called before this function. + */ +int PCU_Comm_Init_2(pcu_t* pcu) +{ + if (pcu->state != pcu_state_uninit) + reel_fail("nested calls to Comm_Init"); + pcu_mpi_init(MPI_COMM_WORLD,&(pcu->mpi)); + pcu_make_msg(&(pcu->pmsg)); + pcu->state = pcu_state_init; + /* turn ordering on by default, call + PCU_Comm_Order(false) after PCU_Comm_Init + to disable this */ + PCU_Comm_Order_2(pcu, true); + return PCU_SUCCESS; +} + +/** \brief Frees all PCU library structures. + \details This function must be called by all MPI processes after all + other calls to PCU, and before calling MPI_Finalize. + */ +int PCU_Comm_Free_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Free called before Comm_Init"); + if (pcu->pmsg.order) + pcu_order_free(pcu->pmsg.order); + pcu_free_msg(&(pcu->pmsg)); + pcu_mpi_finalize(&(pcu->mpi)); + pcu->state = pcu_state_uninit; + return PCU_SUCCESS; +} + +/** \brief Returns the communication rank of the calling thread. + \details when called from a non-threaded MPI process, this function is + equivalent to MPI_Comm_rank(MPI_COMM_WORLD,rank). + + Ranks are consecutive from 0 to \f$pt-1\f$ for a program with + \f$p\f$ processes and \f$t\f$ threads per process. + Ranks are contiguous within a process, so that the \f$t\f$ threads in process + \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. + */ +int PCU_Comm_Self_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Self called before Comm_Init"); + return pcu_mpi_rank(&(pcu->mpi)); +} + +/** \brief Returns the number of threads in the program. + \details when called from a non-threaded MPI process, this function is + equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). + */ +int PCU_Comm_Peers_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Peers called before Comm_Init"); + return pcu_mpi_size(&(pcu->mpi)); +} + +/** \brief Begins a PCU communication phase. + \details This function must be called by all threads in the MPI job + at the beginning of each phase of communication. + After calling this function, each thread may call functions like + PCU_Comm_Pack or PCU_Comm_Write. +*/ +void PCU_Comm_Begin_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Begin called before Comm_Init"); + pcu_msg_start(&(pcu->mpi), get_msg(pcu)); +} + +/** \brief Packs data to be sent to \a to_rank. + \details This function appends the block of \a size bytes starting + at \a data to the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Pack called before Comm_Init"); + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) + reel_fail("Invalid rank in Comm_Pack"); + if ( size > (size_t)INT_MAX ) { + fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); + abort(); + } + memcpy(pcu_msg_pack(get_msg(pcu),to_rank,size),data,size); + return PCU_SUCCESS; +} + +/** \brief Sends all buffers for this communication phase. + \details This function should be called by all threads in the MPI job + after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls + to PCU_Comm_Listen or PCU_Comm_Read. + All buffers from this thread are sent out and receiving + may begin after this call. + */ +int PCU_Comm_Send_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Send called before Comm_Init"); + pcu_msg_send(&(pcu->mpi), get_msg(pcu)); + return PCU_SUCCESS; +} + +/** \brief Tries to receive a buffer for this communication phase. + \details Either this function or PCU_Comm_Read should be called at least + once by all threads during the communication phase, after PCU_Comm_Send + is called. The result will be false if and only if the communication phase + is over and there are no more buffers to receive. + Otherwise, a buffer was received. + Its contents are retrievable through PCU_Comm_Unpack, and its metadata through + PCU_Comm_Sender and PCU_Comm_Received. + Users should unpack all data from this buffer before calling this function + again, because the previously received buffer is destroyed by the call. + */ +bool PCU_Comm_Listen_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Listen called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_receive(&(pcu->mpi), m->order, m); + return pcu_msg_receive(&(pcu->mpi), m); +} + +/** \brief Returns in * \a from_rank the sender of the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + */ +int PCU_Comm_Sender_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Sender called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_received_from(m->order); + return pcu_msg_received_from(m); +} + +/** \brief Returns true if the current received buffer has been unpacked. + \details This function should be called after a successful PCU_Comm_Listen. + */ +bool PCU_Comm_Unpacked_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Unpacked called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_unpacked(m->order); + return pcu_msg_unpacked(m); +} + +/** \brief Unpacks a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + \a data must point to a block of memory of at least \a size bytes, into + which the next \a size bytes of the current received buffer will be written. + Subsequent calls will begin unpacking where this call left off, + so that the entire received buffer can be unpacked by a sequence of calls to + this function. + Users must ensure that there remain \a size bytes to be unpacked, + PCU_Comm_Unpacked can help with this. + */ +int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Unpack called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + memcpy(data,pcu_order_unpack(m->order,size),size); + else + memcpy(data,pcu_msg_unpack(m,size),size); + return PCU_SUCCESS; +} + +void PCU_Comm_Order_2(pcu_t* pcu, bool on) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Order called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (on && (!m->order)) + m->order = pcu_order_new(); + if ((!on) && m->order) { + pcu_order_free(m->order); + m->order = NULL; + } +} + +/** \brief Blocking barrier over all threads. */ +void PCU_Barrier_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Barrier called before Comm_Init"); + pcu_barrier(&(pcu->mpi), &(get_msg(pcu)->coll)); +} + +/** \brief Performs an Allreduce sum of double arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n doubles. + After this call, p[i] will contain the sum of all p[i]'s + given by each rank. + */ +void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_Doubles called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_doubles,p,n*sizeof(double)); +} + +double PCU_Add_Double_2(pcu_t* pcu, double x) +{ + double a[1]; + a[0] = x; + PCU_Add_Doubles_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce minimum of double arrays. + */ +void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Min_Doubles called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_doubles,p,n*sizeof(double)); +} + +double PCU_Min_Double_2(pcu_t* pcu, double x) +{ + double a[1]; + a[0] = x; + PCU_Min_Doubles_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce maximum of double arrays. + */ +void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_Doubles called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_doubles,p,n*sizeof(double)); +} + +double PCU_Max_Double_2(pcu_t* pcu, double x) +{ + double a[1]; + a[0] = x; + PCU_Max_Doubles_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce sum of integers + */ +void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_Ints called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); +} + +int PCU_Add_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Add_Ints_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce sum of long integers + */ +void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_Longs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); +} + +long PCU_Add_Long_2(pcu_t* pcu, long x) +{ + long a[1]; + a[0] = x; + PCU_Add_Longs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce sum of size_t unsigned integers + */ +void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Add_SizeTs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_sizets,p,n*sizeof(size_t)); +} + +size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x) +{ + size_t a[1]; + a[0] = x; + PCU_Add_SizeTs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce minimum of size_t unsigned integers + */ +void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Min_SizeTs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_sizets,p,n*sizeof(size_t)); +} + +size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x) +{ + size_t a[1]; + a[0] = x; + PCU_Min_SizeTs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce maximum of size_t unsigned integers + */ +void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_SizeTs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_sizets,p,n*sizeof(size_t)); +} + +size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x) +{ + size_t a[1]; + a[0] = x; + PCU_Max_SizeTs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an exclusive prefix sum of integer arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n integers. + After this call, p[i] will contain the sum of all p[i]'s + given by ranks lower than the calling rank. + */ +void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Exscan_Ints called before Comm_Init"); + int* originals; + NOTO_MALLOC(originals,n); + for (size_t i=0; i < n; ++i) + originals[i] = p[i]; + pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); + //convert inclusive scan to exclusive + for (size_t i=0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} + +int PCU_Exscan_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Exscan_Ints_2(pcu, a, 1); + return a[0]; +} + +/** \brief See PCU_Exscan_Ints */ +void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Exscan_Longs called before Comm_Init"); + long* originals; + NOTO_MALLOC(originals,n); + for (size_t i=0; i < n; ++i) + originals[i] = p[i]; + pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); + //convert inclusive scan to exclusive + for (size_t i=0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} + +long PCU_Exscan_Long_2(pcu_t* pcu, long x) +{ + long a[1]; + a[0] = x; + PCU_Exscan_Longs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce minimum of int arrays. + */ +void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Min_Ints called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_ints,p,n*sizeof(int)); +} + +int PCU_Min_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Min_Ints_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs an Allreduce maximum of int arrays. + */ +void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_Ints called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_ints,p,n*sizeof(int)); +} + +int PCU_Max_Int_2(pcu_t* pcu, int x) +{ + int a[1]; + a[0] = x; + PCU_Max_Ints_2(pcu, a, 1); + return a[0]; +} +/** \brief Performs an Allreduce maximum of long arrays. + */ +void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Max_Longs called before Comm_Init"); + pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_longs,p,n*sizeof(long)); +} + +long PCU_Max_Long_2(pcu_t* pcu, long x) +{ + long a[1]; + a[0] = x; + PCU_Max_Longs_2(pcu, a, 1); + return a[0]; +} + +/** \brief Performs a parallel logical OR reduction + */ +int PCU_Or_2(pcu_t* pcu, int c) +{ + return PCU_Max_Int_2(pcu, c); +} + +/** \brief Performs a parallel logical AND reduction + */ +int PCU_And_2(pcu_t* pcu, int c) +{ + return PCU_Min_Int_2(pcu, c); +} + +/** \brief Returns the unique rank of the calling process. + */ +int PCU_Proc_Self_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Proc_Self called before Comm_Init"); + return pcu_mpi_rank(&(pcu->mpi)); +} + +/** \brief Returns the number of processes. + */ +int PCU_Proc_Peers_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Proc_Peers called before Comm_Init"); + return pcu_mpi_size(&(pcu->mpi)); +} + +/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. + */ +int PCU_Comm_Rank_2(pcu_t* pcu, int* rank) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Rank called before Comm_Init"); + *rank = pcu_mpi_rank(&(pcu->mpi)); + return PCU_SUCCESS; +} + +/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ +int PCU_Comm_Size_2(pcu_t* pcu, int* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Size called before Comm_Init"); + *size = pcu_mpi_size(&(pcu->mpi)); + return PCU_SUCCESS; +} + +/** \brief Returns true iff PCU has been initialized */ +bool PCU_Comm_Initialized_2(pcu_t* pcu) +{ + return pcu->state == pcu_state_init; +} + +/** \brief Deprecated, see PCU_Comm_Begin. + */ +int PCU_Comm_Start_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Start called before Comm_Init"); + pcu_msg_start(&(pcu->mpi),get_msg(pcu)); + return PCU_SUCCESS; +} + +/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. + \details Returns the size of the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Packed called before Comm_Init"); + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) + reel_fail("Invalid rank in Comm_Packed"); + *size = pcu_msg_packed(get_msg(pcu),to_rank); + return PCU_SUCCESS; +} + +/** \brief Packs a message to be sent to \a to_rank. + \details This function packs a message into the buffer being sent + to \a to_rank. + Messages packed by this function can be received using the function + PCU_Comm_Read. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + If this function is used, PCU_Comm_Pack should not be used. + */ +int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Write called before Comm_Init"); + if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) + reel_fail("Invalid rank in Comm_Write"); + pcu_msg* msg = get_msg(pcu); + PCU_MSG_PACK(msg,to_rank,size); + memcpy(pcu_msg_pack(msg,to_rank,size),data,size); + return PCU_SUCCESS; +} + +/** \brief Convenience wrapper over Listen and Unpacked */ +bool PCU_Comm_Receive_2(pcu_t* pcu) +{ + while (PCU_Comm_Unpacked_2(pcu)) + if (!PCU_Comm_Listen_2(pcu)) + return false; + return true; +} + +/** \brief Receives a message for this communication phase. + \details This function tries to receive a message packed by + PCU_Comm_Write. + If a the communication phase is over and there are no more + messages to receive, this function returns false. + Otherwise, * \a from_rank will be the rank which sent the message, + * \a data will point to the start of the message data, and + * \a size will be the number of bytes of message data. + If this function is used, PCU_Comm_Receive should not be used. + Note that the address * \a data points into a PCU buffer, so + it is strongly recommended that this data be read and not modified. + */ +bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Read called before Comm_Init"); + if (!PCU_Comm_Receive_2(pcu)) + return false; + *from_rank = PCU_Comm_Sender_2(pcu); + PCU_Comm_Unpack_2(pcu, size, sizeof(*size)); + *data = PCU_Comm_Extract_2(pcu, *size); + return true; +} + +static void safe_mkdir(const char* path, mode_t mode) +{ + int err; + errno = 0; + err = mkdir(path, mode); + if (err != 0 && errno != EEXIST) + reel_fail("PCU: could not create directory \"%s\"\n", path); +} + +static void append(char* s, size_t size, const char* format, ...) +{ + int len = strlen(s); + va_list ap; + va_start(ap, format); + vsnprintf(s + len, size - len, format, ap); + va_end(ap); +} + + +void PCU_Debug_Open_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Debug_Open called before Comm_Init"); + + const int fanout = 2048; + const int bufsize = 1024; + char* path = noto_malloc(bufsize); + path[0] = '\0'; + if (PCU_Comm_Peers_2(pcu) > fanout) { + mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; + strcpy(path, "debug/"); + safe_mkdir(path, dir_perm); + int self = PCU_Comm_Self_2(pcu); + append(path, bufsize, "%d/", self / fanout); + if (self % fanout == 0) + safe_mkdir(path, dir_perm); + PCU_Barrier_2(pcu); + } + + append(path,bufsize, "%s", "debug"); + pcu_msg* msg = get_msg(pcu); + if ( ! msg->file) + msg->file = pcu_open_parallel(path,"txt"); + noto_free(path); +} + +void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list ap) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Debug_Print called before Comm_Init"); + pcu_msg* msg = get_msg(pcu); + if ( ! msg->file) + return; //Print is a no-op if no file is open + vfprintf(msg->file,format,ap); + fflush(msg->file); +} +/** \brief like fprintf, contents go to debugN.txt */ +void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) +{ + va_list arglist; + va_start(arglist,format); + PCU_Debug_Printv_2(pcu, format, arglist); + va_end(arglist); +} + +/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ +int PCU_Comm_From_2(pcu_t* pcu, int* from_rank) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_From called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + *from_rank = pcu_order_received_from(m->order); + else + *from_rank = pcu_msg_received_from(m); + return PCU_SUCCESS; +} + +/** \brief Returns in * \a size the bytes in the current received buffer + \details This function should be called after a successful PCU_Comm_Receive. + The size returned will be the total received size regardless of how + much unpacking has been done. + */ +int PCU_Comm_Received_2(pcu_t* pcu, size_t* size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Received called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + *size = pcu_order_received_size(m->order); + else + *size = pcu_msg_received_size(m); + return PCU_SUCCESS; +} + +/** \brief Extracts a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Receive. + The next \a size bytes of the current received buffer are unpacked, + and an internal pointer to that data is returned. + The returned pointer must not be freed by the user. + */ +void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Comm_Extract called before Comm_Init"); + pcu_msg* m = get_msg(pcu); + if (m->order) + return pcu_order_unpack(m->order,size); + return pcu_msg_unpack(m,size); +} + +/** \brief Reinitializes PCU with a new MPI communicator. + \details All of PCU's logic is based off two duplicates + of this communicator, so you can safely get PCU to act + on sub-groups of processes using this function. + This call should be collective over all processes + in the previous communicator. This is a very heavy weight function + and should be used sparingly. + */ +void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Switch_Comm called before Comm_Init"); + int result; + MPI_Comm_compare(new_comm, pcu->mpi.original_comm, &result); + if (result != MPI_IDENT) { + pcu_mpi_finalize(&(pcu->mpi)); + pcu_mpi_init(new_comm,&(pcu->mpi)); + } +} + +/** \brief Return the current MPI communicator + \details Returns the communicator given to the + most recent PCU_Switch_Comm call, or MPI_COMM_WORLD + otherwise. + */ +MPI_Comm PCU_Get_Comm_2(pcu_t* pcu) +{ + if (pcu->state == pcu_state_uninit) + reel_fail("Get_Comm called before Comm_Init"); + return pcu->mpi.original_comm; +} + +/** \brief Return the time in seconds since some time in the past + */ +double PCU_Time_2(pcu_t* pcu) +{ + return MPI_Wtime(); +} + +void PCU_Protect_2(pcu_t* pcu) +{ + reel_protect(); +} From ef021f6b9f104a77eecd2c00c6de9e5c58cb3376 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 14:21:54 -0400 Subject: [PATCH 022/141] add PCU2.h to install files --- pcu/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 8c5598168..1bee4f701 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -28,6 +28,7 @@ set(SOURCES # Package headers set(HEADERS PCU.h + PCU2.h pcu_io.h pcu_util.h reel/reel.h From 270d44eaeb92b1d2996b28876eaf938f149fc060 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 15:03:06 -0400 Subject: [PATCH 023/141] fix compiler warnings --- pcu/pcu.c | 4 ++-- pcu/pcu2.c | 4 ++++ pcu/pcu_coll.c | 20 ++++++++++++++++++++ pcu/pcu_pmpi.c | 8 ++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pcu/pcu.c b/pcu/pcu.c index d4679fc86..1e82ff272 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.c @@ -53,7 +53,7 @@ static pcu_t global_pcu = {.state=pcu_state_uninit}; */ int PCU_Comm_Init(void) { - PCU_Comm_Init_2(&global_pcu); + return PCU_Comm_Init_2(&global_pcu); } /** \brief Frees all PCU library structures. @@ -504,7 +504,7 @@ void PCU_Switch_Comm(MPI_Comm new_comm) */ MPI_Comm PCU_Get_Comm(void) { - PCU_Get_Comm_2(&global_pcu); + return PCU_Get_Comm_2(&global_pcu); } /** \brief Return the time in seconds since some time in the past diff --git a/pcu/pcu2.c b/pcu/pcu2.c index 001ac9ed2..03b4db62d 100644 --- a/pcu/pcu2.c +++ b/pcu/pcu2.c @@ -774,10 +774,14 @@ MPI_Comm PCU_Get_Comm_2(pcu_t* pcu) */ double PCU_Time_2(pcu_t* pcu) { + // silence warning + (void)pcu; return MPI_Wtime(); } void PCU_Protect_2(pcu_t* pcu) { + // silence warning + (void)pcu; reel_protect(); } diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 225113b13..a3ad4e497 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -170,6 +170,8 @@ static bool end_coll_step(pcu_mpi_t* mpi, pcu_coll* c) void pcu_make_coll(pcu_mpi_t* mpi, pcu_coll* c, pcu_pattern* p, pcu_merge* m) { + // silence warning + (void)mpi; c->pattern = p; c->merge = m; } @@ -225,6 +227,8 @@ bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) static int reduce_begin_bit(pcu_mpi_t* mpi) { + // silence warning + (void)mpi; return 1; } @@ -252,6 +256,8 @@ static int reduce_action(pcu_mpi_t* mpi, int bit) static int reduce_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit << 1; } @@ -280,6 +286,8 @@ static int bcast_begin_bit(pcu_mpi_t* mpi) static bool bcast_end_bit(pcu_mpi_t * mpi, int bit) { + // silence warning + (void)mpi; return bit == 0; } @@ -299,6 +307,8 @@ static int bcast_action(pcu_mpi_t* mpi, int bit) static int bcast_shift(pcu_mpi_t * mpi, int bit) { + // silence warning + (void)mpi; return bit >> 1; } @@ -320,6 +330,8 @@ static pcu_pattern bcast = static int scan_up_begin_bit(pcu_mpi_t* mpi) { + // silence warning + (void)mpi; return 1; } @@ -373,6 +385,8 @@ static int scan_up_peer(pcu_mpi_t* mpi, int bit) static int scan_up_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit << 1; } @@ -392,6 +406,8 @@ static int scan_down_begin_bit(pcu_mpi_t* mpi) static bool scan_down_end_bit(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit == 1; } @@ -442,6 +458,8 @@ static int scan_down_peer(pcu_mpi_t * mpi, int bit) static int scan_down_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit >> 1; } @@ -507,6 +525,8 @@ bool pcu_barrier_done(pcu_mpi_t* mpi, pcu_coll* c) void pcu_barrier(pcu_mpi_t* mpi, pcu_coll* c) { + // silence warning + (void)mpi; pcu_begin_barrier(mpi, c); while( ! pcu_barrier_done(mpi, c)); } diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index a4280dace..5bfd3f581 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -48,6 +48,8 @@ void pcu_pmpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { + // silence warning + (void)self; if( m->buffer.size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR PCU message size exceeds INT_MAX... exiting\n"); abort(); @@ -64,6 +66,8 @@ void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) { + // silence warning + (void)self; int flag; MPI_Test(&(m->request),&flag,MPI_STATUS_IGNORE); return flag; @@ -71,11 +75,15 @@ bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) bool pcu_pmpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { + // silence warning + (void)self; return pcu_pmpi_receive2(self, m,0,comm); } bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { + // silence warning + (void)self; MPI_Status status; int flag; MPI_Iprobe(m->peer,tag,comm,&flag,&status); From 62bfb691b9879994d050dd79d6e6a3fe0918125f Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Wed, 29 Mar 2023 22:11:41 -0400 Subject: [PATCH 024/141] constify mpi functions --- pcu/pcu_mpi.c | 17 ++++++----------- pcu/pcu_mpi.h | 10 +++++----- pcu/pcu_pmpi.c | 18 +++++++++--------- pcu/pcu_pmpi.h | 10 +++++----- 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index 40236ef95..5681d0e90 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -21,41 +21,36 @@ void pcu_free_message(pcu_message* m) pcu_free_buffer(&(m->buffer)); } -int pcu_mpi_size(pcu_mpi_t* self) +int pcu_mpi_size(const pcu_mpi_t* self) { return pcu_pmpi_size(self); } -int pcu_mpi_rank(pcu_mpi_t* self) +int pcu_mpi_rank(const pcu_mpi_t* self) { return pcu_pmpi_rank(self); } -static void check_rank(pcu_mpi_t* self, int rank) +static void check_rank(const pcu_mpi_t* self, int rank) { (void)rank; PCU_ALWAYS_ASSERT(0 <= rank); PCU_ALWAYS_ASSERT(rank < pcu_mpi_size(self)); } -void pcu_mpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +void pcu_mpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { check_rank(self, m->peer); - // verify that the communicator is one of the user or collective communicators - //int user_result, coll_result; - //MPI_Comm_compare(comm, self->user_comm, &user_result); - //MPI_Comm_compare(comm, self->coll_comm, &coll_result); - //PCU_ALWAYS_ASSERT(user_result == MPI_IDENT || coll_result == MPI_IDENT); PCU_ALWAYS_ASSERT(comm == self->user_comm || comm == self->coll_comm); pcu_pmpi_send(self, m, comm); } -bool pcu_mpi_done(pcu_mpi_t* self, pcu_message* m) +bool pcu_mpi_done(const pcu_mpi_t* self, pcu_message* m) { return pcu_pmpi_done(self, m); } -bool pcu_mpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +bool pcu_mpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { if (m->peer != MPI_ANY_SOURCE) check_rank(self, m->peer); diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 9c0fad4f4..fb55ea275 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -32,11 +32,11 @@ typedef struct int size; } pcu_mpi_t; -int pcu_mpi_size(pcu_mpi_t*); -int pcu_mpi_rank(pcu_mpi_t*); -void pcu_mpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_mpi_done(pcu_mpi_t*, pcu_message* m); -bool pcu_mpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +int pcu_mpi_size(const pcu_mpi_t*); +int pcu_mpi_rank(const pcu_mpi_t*); +void pcu_mpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_mpi_done(const pcu_mpi_t*, pcu_message* m); +bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); void pcu_mpi_finalize(pcu_mpi_t* mpi); diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 5bfd3f581..1db376d2c 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -13,8 +13,8 @@ #include #include -void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_receive2(pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); +void pcu_pmpi_send2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); +bool pcu_pmpi_receive2(const pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* self) { @@ -31,22 +31,22 @@ void pcu_pmpi_finalize(pcu_mpi_t* self) MPI_Comm_free(&(self->coll_comm)); } -int pcu_pmpi_size(pcu_mpi_t* self) +int pcu_pmpi_size(const pcu_mpi_t* self) { return self->size; } -int pcu_pmpi_rank(pcu_mpi_t* self) +int pcu_pmpi_rank(const pcu_mpi_t* self) { return self->rank; } -void pcu_pmpi_send(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +void pcu_pmpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { pcu_pmpi_send2(self, m,0,comm); } -void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) +void pcu_pmpi_send2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { // silence warning (void)self; @@ -64,7 +64,7 @@ void pcu_pmpi_send2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) &(m->request)); } -bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) +bool pcu_pmpi_done(const pcu_mpi_t* self, pcu_message* m) { // silence warning (void)self; @@ -73,14 +73,14 @@ bool pcu_pmpi_done(pcu_mpi_t* self, pcu_message* m) return flag; } -bool pcu_pmpi_receive(pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +bool pcu_pmpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { // silence warning (void)self; return pcu_pmpi_receive2(self, m,0,comm); } -bool pcu_pmpi_receive2(pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) +bool pcu_pmpi_receive2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { // silence warning (void)self; diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index 36507421d..d6dd08dae 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -16,10 +16,10 @@ void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* mpi); void pcu_pmpi_finalize(pcu_mpi_t* m); -int pcu_pmpi_size(pcu_mpi_t* self); -int pcu_pmpi_rank(pcu_mpi_t* self); -void pcu_pmpi_send(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_done(pcu_mpi_t*, pcu_message* m); +int pcu_pmpi_size(const pcu_mpi_t* self); +int pcu_pmpi_rank(const pcu_mpi_t* self); +void pcu_pmpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_pmpi_done(const pcu_mpi_t*, pcu_message* m); #endif From 173c7ec93b5c71414579816fc4d304ce039cc2b4 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 02:11:09 -0400 Subject: [PATCH 025/141] first pass of PCU with making use of a high level object --- pcu/CMakeLists.txt | 9 +- pcu/PCU.h | 12 +- pcu/PCU2.h | 137 ------- pcu/PCUObj.cc | 276 +++++++++++++++ pcu/PCUObj.h | 101 ++++++ pcu/noto/noto_malloc.h | 23 +- pcu/{pcu.c => pcu.cc} | 466 +++++++++++++----------- pcu/pcu2.c | 787 ----------------------------------------- pcu/pcu_buffer.h | 6 + pcu/pcu_byteorder.h | 8 + pcu/pcu_coll.c | 1 + pcu/pcu_coll.h | 7 + pcu/pcu_defines.h | 13 + pcu/pcu_mem.c | 13 +- pcu/pcu_mem.h | 13 + pcu/pcu_mpi.h | 12 +- pcu/pcu_msg.h | 7 + pcu/pcu_order.h | 6 + pcu/pcu_pmpi.h | 27 +- pcu/pcu_util.h | 36 +- pcu/pkg_tribits.cmake | 39 +- 21 files changed, 798 insertions(+), 1201 deletions(-) delete mode 100644 pcu/PCU2.h create mode 100644 pcu/PCUObj.cc create mode 100644 pcu/PCUObj.h rename pcu/{pcu.c => pcu.cc} (51%) delete mode 100644 pcu/pcu2.c create mode 100644 pcu/pcu_defines.h create mode 100644 pcu/pcu_mem.h diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 1bee4f701..3a3272220 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,8 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.c - pcu2.c + pcu.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -23,15 +22,15 @@ set(SOURCES pcu_util.c noto/noto_malloc.c reel/reel.c -) + PCUObj.cc) # Package headers set(HEADERS PCU.h - PCU2.h - pcu_io.h + pcu_io.h pcu_util.h reel/reel.h + pcu_defines.h ) # Add the pcu library diff --git a/pcu/PCU.h b/pcu/PCU.h index f9a53362e..02ff1255a 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -9,10 +9,7 @@ *******************************************************************************/ #ifndef PCU_H #define PCU_H - -#define PCU_SUCCESS 0 -#define PCU_FAILURE -1 - +#include "pcu_defines.h" #include #ifdef __cplusplus @@ -94,13 +91,8 @@ bool PCU_Comm_Read(int* from_rank, void** data, size_t* size); /*Debug file I/O API*/ void PCU_Debug_Open(void); -#ifdef __GNUC__ -void PCU_Debug_Print(const char* format, ...) - __attribute__((format(printf,1,2))); -#else -void PCU_Debug_Print(const char* format, ...); -#endif +void PCU_Debug_Print(const char* format, ...) PCU_FORMAT_ATTRIBUTE(1,2); /*lesser-used APIs*/ bool PCU_Comm_Initialized(void); int PCU_Comm_Packed(int to_rank, size_t* size); diff --git a/pcu/PCU2.h b/pcu/PCU2.h deleted file mode 100644 index 3601ff029..000000000 --- a/pcu/PCU2.h +++ /dev/null @@ -1,137 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -#ifndef PCU2_H -#define PCU2_H - -#define PCU_SUCCESS 0 -#define PCU_FAILURE -1 - -#include - -#ifdef __cplusplus -#include -#include -extern "C" { -#else -#include -#include -#include -#endif - -// FOR pcu_state_type ... use opaque ptr instead? -#include "pcu_msg.h" - -typedef enum { pcu_state_uninit, pcu_state_init } pcu_state_enum ; -typedef struct { - pcu_state_enum state; - pcu_msg pmsg; - pcu_mpi_t mpi; -} pcu_t; - -/*library init/finalize*/ -int PCU_Comm_Init_2(pcu_t* pcu); -int PCU_Comm_Free_2(pcu_t* pcu); - -/*rank/size functions*/ -int PCU_Comm_Self_2(pcu_t* pcu); -int PCU_Comm_Peers_2(pcu_t* pcu); - -/*recommended message passing API*/ -void PCU_Comm_Begin_2(pcu_t* pcu); -int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size); -int PCU_Comm_Send_2(pcu_t* pcu); -bool PCU_Comm_Receive_2(pcu_t* pcu); -bool PCU_Comm_Listen_2(pcu_t* pcu); -int PCU_Comm_Sender_2(pcu_t* pcu); -bool PCU_Comm_Unpacked_2(pcu_t* pcu); -int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size); - -/*turns deterministic ordering for the - above API on/off*/ -void PCU_Comm_Order_2(pcu_t* pcu, bool on); - -/*collective operations*/ -void PCU_Barrier_2(pcu_t* pcu); -void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n); -double PCU_Add_Double_2(pcu_t* pcu, double x); -void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n); -double PCU_Min_Double_2(pcu_t* pcu, double x); -void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n); -double PCU_Max_Double_2(pcu_t* pcu, double x); -void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Add_Int_2(pcu_t* pcu, int x); -void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n); -long PCU_Add_Long_2(pcu_t* pcu, long x); -void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Exscan_Int_2(pcu_t* pcu, int x); -void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n); -long PCU_Exscan_Long_2(pcu_t* pcu, long x); -void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); -size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x); -void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); -size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x); -void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n); -size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x); -void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Min_Int_2(pcu_t* pcu, int x); -void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n); -int PCU_Max_Int_2(pcu_t* pcu, int x); -void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n); -long PCU_Max_Long_2(pcu_t* pcu, long x); -int PCU_Or_2(pcu_t* pcu, int c); -int PCU_And_2(pcu_t* pcu, int c); - -/*process-level self/peers (mpi wrappers)*/ -int PCU_Proc_Self_2(pcu_t* pcu); -int PCU_Proc_Peers_2(pcu_t* pcu); - -/*IPComMan replacement API*/ -int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size); -bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size); - -/*Debug file I/O API*/ -void PCU_Debug_Open_2(pcu_t* pcu); -void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list args); -#ifdef __GNUC__ -void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) - __attribute__((format(printf,2,3))); -#else -void PCU_Debug_Print(pcu_t* pcu, const char* format, ...); -#endif - -/*lesser-used APIs*/ -bool PCU_Comm_Initialized_2(pcu_t* pcu); -int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size); -int PCU_Comm_From_2(pcu_t* pcu, int* from_rank); -int PCU_Comm_Received_2(pcu_t* pcu, size_t* size); -void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size); -int PCU_Comm_Rank_2(pcu_t* pcu, int* rank); -int PCU_Comm_Size_2(pcu_t* pcu, int* size); - -int PCU_Comm_Start_2(pcu_t* pcu); - -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm_2(pcu_t* pcu); - -/*stack trace helpers using GNU/Linux*/ -void PCU_Protect_2(pcu_t* pcu); - -/*MPI_Wtime_() equivalent*/ -double PCU_Time_2(pcu_t* pcu); - -/*Memory usage*/ -double PCU_GetMem_2(pcu_t* pcu); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc new file mode 100644 index 000000000..a6a3f9b76 --- /dev/null +++ b/pcu/PCUObj.cc @@ -0,0 +1,276 @@ +#include "PCUObj.h" +#include "noto_malloc.h" +#include "pcu_mem.h" +#include "pcu_mpi.h" +#include "pcu_msg.h" +#include "pcu_order.h" +#include "reel.h" +#include +#include /*using POSIX mkdir call for SMB "foo/" path*/ +namespace pcu { + +int PCU::Peers() const noexcept { return pcu_mpi_size(mpi_); } +int PCU::Self() const noexcept { return pcu_mpi_rank(mpi_); } +void PCU::Begin() noexcept { pcu_msg_start(mpi_, msg_); } +int PCU::Pack(int to_rank, const void *data, size_t size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Pack"); + if (size > (size_t)INT_MAX) { + fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds " + "INT_MAX... exiting\n"); + abort(); + } + memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); + return PCU_SUCCESS; +} +int PCU::Send() noexcept { + pcu_msg_send(mpi_, msg_); + return PCU_SUCCESS; +} +bool PCU::Receive(void) noexcept { + while (Unpacked()) + if (!Listen()) + return false; + return true; +} +bool PCU::Listen() noexcept { + if (msg_->order) + return pcu_order_receive(mpi_, msg_->order, msg_); + return pcu_msg_receive(mpi_, msg_); +} +int PCU::Sender() noexcept { + if (msg_->order) + return pcu_order_received_from(msg_->order); + return pcu_msg_received_from(msg_); +} +bool PCU::Unpacked() noexcept { + if (msg_->order) + return pcu_order_unpacked(msg_->order); + return pcu_msg_unpacked(msg_); +} +int PCU::Unpack(void *data, size_t size) noexcept { + if (msg_->order) + memcpy(data, pcu_order_unpack(msg_->order, size), size); + else + memcpy(data, pcu_msg_unpack(msg_, size), size); + return PCU_SUCCESS; +} +int PCU::Write(int to_rank, const void *data, size_t size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Write"); + PCU_MSG_PACK(msg_, to_rank, size); + memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); + return PCU_SUCCESS; +} +bool PCU::Read(int *from_rank, void **data, size_t *size) noexcept { + if (!Receive()) + return false; + *from_rank = Sender(); + Unpack(size, sizeof(*size)); + *data = Extract(*size); + return true; +} +void PCU::Order(bool on) { + if (on && (!msg_->order)) + msg_->order = pcu_order_new(); + if ((!on) && msg_->order) { + pcu_order_free(msg_->order); + msg_->order = NULL; + } +} +void PCU::Barrier() { pcu_barrier(mpi_, &(msg_->coll)); } +int PCU::Or(int c) noexcept { return Max(c); } +int PCU::And(int c) noexcept { return Min(c); } +int PCU::Packed(int to_rank, size_t *size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Packed"); + *size = pcu_msg_packed(msg_, to_rank); + return PCU_SUCCESS; +} +int PCU::From(int *from_rank) noexcept { + if (msg_->order) + *from_rank = pcu_order_received_from(msg_->order); + else + *from_rank = pcu_msg_received_from(msg_); + return PCU_SUCCESS; +} +int PCU::Received(size_t *size) noexcept { + if (msg_->order) + *size = pcu_order_received_size(msg_->order); + else + *size = pcu_msg_received_size(msg_); + return PCU_SUCCESS; +} +void *PCU::Extract(size_t size) noexcept { + if (msg_->order) + return pcu_order_unpack(msg_->order, size); + return pcu_msg_unpack(msg_, size); +} + +static void safe_mkdir(const char *path, mode_t mode) { + int err; + errno = 0; + err = mkdir(path, mode); + if (err != 0 && errno != EEXIST) + reel_fail("PCU: could not create directory \"%s\"\n", path); +} + +static void append(char *s, size_t size, const char *format, ...) { + int len = strlen(s); + va_list ap; + va_start(ap, format); + vsnprintf(s + len, size - len, format, ap); + va_end(ap); +} + +void PCU::DebugOpen() noexcept { + const int fanout = 2048; + const int bufsize = 1024; + char *path = (char *)noto_malloc(bufsize); + path[0] = '\0'; + if (Peers() > fanout) { + mode_t const dir_perm = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; + strcpy(path, "debug/"); + safe_mkdir(path, dir_perm); + int self = Self(); + append(path, bufsize, "%d/", self / fanout); + if (self % fanout == 0) + safe_mkdir(path, dir_perm); + Barrier(); + } + + append(path, bufsize, "%s", "debug"); + if (!msg_->file) + msg_->file = pcu_open_parallel(path, "txt"); + noto_free(path); +} + +double GetMem(void) noexcept { return pcu_get_mem(); } +void Protect(void) noexcept { reel_protect(); } +double Time(void) noexcept { return MPI_Wtime(); } + +void PCU::DebugPrint(const char *format, ...) noexcept { + va_list args; + va_start(args, format); + DebugPrint(format, args); + va_end(args); +} +void PCU::DebugPrint(const char *format, va_list args) noexcept { + if (!msg_->file) + return; // Print is a no-op if no file is open + vfprintf(msg_->file, format, args); + fflush(msg_->file); +} +PCU::PCU(MPI_Comm comm) { + mpi_ = new pcu_mpi_t; + msg_ = new pcu_msg; + pcu_mpi_init(comm, mpi_); + pcu_make_msg(msg_); + /* turn ordering on by default, call + PCU_Comm_Order(false) after PCU_Comm_Init + to disable this */ + Order(true); +} +PCU::~PCU() noexcept { + delete mpi_; + delete msg_; +} +PCU::PCU(PCU &&other) noexcept { + std::swap(mpi_, other.mpi_); + std::swap(msg_, other.msg_); +} +PCU &PCU::operator=(PCU && other) noexcept { + std::swap(mpi_, other.mpi_); + std::swap(msg_, other.msg_); + return *this; +} +MPI_Comm PCU::GetMPIComm() const noexcept { return mpi_->original_comm; } + +/* template implementations */ +template void PCU::Add(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] += b[i]; + }, + p, n * sizeof(T)); +} +template T PCU::Add(T p) noexcept { + Add(&p, 1); + return p; +} +template void PCU::Min(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] = std::min(a[i], b[i]); + }, + p, n * sizeof(T)); +} +template T PCU::Min(T p) noexcept { + Min(&p, 1); + return p; +} +template void PCU::Max(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] = std::max(a[i], b[i]); + }, + p, n * sizeof(T)); +} +template T PCU::Max(T p) noexcept { + Max(&p, 1); + return p; +} +template void PCU::Exscan(T *p, size_t n) noexcept { + auto *originals = (T *)noto_malloc(sizeof(T) * n); + for (size_t i = 0; i < n; ++i) + originals[i] = p[i]; + pcu_scan( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] += b[i]; + }, + p, n * sizeof(T)); + // convert inclusive scan to exclusive + for (size_t i = 0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} +template T PCU::Exscan(T p) noexcept { + Exscan(&p, 1); + return p; +} +#define PCU_EXPL_INST_DECL(T) \ + template void PCU::Add(T * p, size_t n) noexcept; \ + template T PCU::Add(T p) noexcept; \ + template void PCU::Min(T * p, size_t n) noexcept; \ + template T PCU::Min(T p) noexcept; \ + template void PCU::Max(T * p, size_t n) noexcept; \ + template T PCU::Max(T p) noexcept; \ + template void PCU::Exscan(T * p, size_t n) noexcept; \ + template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL + +} // namespace pcu diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h new file mode 100644 index 000000000..9584f5b6c --- /dev/null +++ b/pcu/PCUObj.h @@ -0,0 +1,101 @@ +#ifndef SCOREC_PCU_PCUOBJ_H +#define SCOREC_PCU_PCUOBJ_H +#include +#include +#include "pcu_defines.h" + +struct pcu_msg_struct; +struct pcu_mpi_struct; + +namespace pcu { +class PCU { +public: + explicit PCU(MPI_Comm comm); + ~PCU() noexcept; + PCU(PCU const &) = delete; + PCU(PCU &&) noexcept; + PCU &operator=(PCU const &) = delete; + PCU &operator=(PCU &&) noexcept; + /** @brief Returns the rank of the current process. + * @return The rank of the current process. + */ + [[nodiscard]] int Self() const noexcept; + /** @brief Returns the number of ranks in the communicator. + * @return The number of ranks in the communicator. + */ + [[nodiscard]] int Peers() const noexcept; + [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; + + /*recommended message passing API*/ + void Begin() noexcept; + int Pack(int to_rank, const void *data, size_t size) noexcept; + int Send() noexcept; + bool Receive() noexcept; + bool Listen() noexcept; + int Sender() noexcept; + bool Unpacked() noexcept; + int Unpack(void *data, size_t size) noexcept; + /*IPComMan replacement API*/ + int Write(int to_rank, const void *data, size_t size) noexcept; + bool Read(int *from_rank, void **data, size_t *size) noexcept; + + /*turns deterministic ordering for the + above API on/off*/ + void Order(bool on); + + /*collective operations*/ + void Barrier(); + template void Add(T *p, size_t n) noexcept; + template [[nodiscard]] T Add(T p) noexcept; + template void Min(T *p, size_t n) noexcept; + template [[nodiscard]] T Min(T p) noexcept; + template void Max(T *p, size_t n) noexcept; + template [[nodiscard]] T Max(T p) noexcept; + template void Exscan(T *p, size_t n) noexcept; + template [[nodiscard]] T Exscan(T p) noexcept; + + /*bitwise operations*/ + [[nodiscard]] int Or(int c) noexcept; + [[nodiscard]] int And(int c) noexcept; + + /*lesser-used APIs*/ + int Packed(int to_rank, size_t *size) noexcept; + int From(int *from_rank) noexcept; + int Received(size_t *size) noexcept; + void *Extract(size_t size) noexcept; + + void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3); + void DebugPrint(const char* format, va_list args) noexcept; + /* Debug functions */ + void DebugOpen() noexcept; + +private: + pcu_msg_struct *msg_; + pcu_mpi_struct *mpi_; +}; +/*stack trace helpers using GNU/Linux*/ +void Protect(void) noexcept; +/*Memory usage*/ +[[nodiscard]] double GetMem(void) noexcept; +/*MPI_Wtime() equivalent*/ +[[nodiscard]] double Time(void) noexcept; + +/* explicit instantiations of template functions */ +#define PCU_EXPL_INST_DECL(T) \ + extern template void PCU::Add(T * p, size_t n) noexcept; \ + extern template T PCU::Add(T p) noexcept; \ + extern template void PCU::Min(T * p, size_t n) noexcept; \ + extern template T PCU::Min(T p) noexcept; \ + extern template void PCU::Max(T * p, size_t n) noexcept; \ + extern template T PCU::Max(T p) noexcept; \ + extern template void PCU::Exscan(T * p, size_t n) noexcept; \ + extern template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL + +} // namespace pcu +#undef PCU_FORMAT_ATTRIBUTE +#endif // SCOREC_PCU_PCUOBJ_H diff --git a/pcu/noto/noto_malloc.h b/pcu/noto/noto_malloc.h index e1f60ed36..74a2c4bac 100644 --- a/pcu/noto/noto_malloc.h +++ b/pcu/noto/noto_malloc.h @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2015 Scientific Computation Research Center, + Copyright 2015 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -10,13 +10,18 @@ #ifndef NOTO_MALLOC_H #define NOTO_MALLOC_H +#ifdef __cplusplus +extern "C" { +#endif + #include -void* noto_malloc(size_t size); -#define NOTO_MALLOC(p,c) ((p)=noto_malloc(sizeof(*(p))*(c))) -void* noto_realloc(void* p, size_t size); -void noto_free(void* p); +void *noto_malloc(size_t size); +#define NOTO_MALLOC(p, c) ((p) = noto_malloc(sizeof(*(p)) * (c))) +void *noto_realloc(void *p, size_t size); +void noto_free(void *p); size_t noto_malloced(void); - +#ifdef __cplusplus +} +#endif #endif - diff --git a/pcu/pcu.c b/pcu/pcu.cc similarity index 51% rename from pcu/pcu.c rename to pcu/pcu.cc index 1e82ff272..3a0337812 100644 --- a/pcu/pcu.c +++ b/pcu/pcu.cc @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2011 Scientific Computation Research Center, + Copyright 2011 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -29,40 +29,35 @@ The API documentation is here: pcu.c */ -#include -#include #include "PCU.h" -#include "PCU2.h" -#include "pcu_msg.h" -#include "pcu_order.h" -#include "noto_malloc.h" +#include "PCUObj.h" #include "reel.h" -#include /*required for mode_t for mkdir on some systems*/ -#include /*using POSIX mkdir call for SMB "foo/" path*/ -#include /* for checking the error from mkdir */ -#include /*INT_MAX*/ -#include /*abort*/ - - -static pcu_t global_pcu = {.state=pcu_state_uninit}; +#include +static pcu::PCU *global_pcu = nullptr; +extern "C" { /** \brief Initializes the PCU library. \details This function must be called by all MPI processes before calling any other PCU functions. MPI_Init or MPI_Init_thread should be called before this function. */ -int PCU_Comm_Init(void) -{ - return PCU_Comm_Init_2(&global_pcu); +int PCU_Comm_Init(void) { + if (global_pcu != nullptr) + reel_fail("nested calls to Comm_Init"); + global_pcu = new pcu::PCU(MPI_COMM_WORLD); + return PCU_SUCCESS; } /** \brief Frees all PCU library structures. \details This function must be called by all MPI processes after all other calls to PCU, and before calling MPI_Finalize. */ -int PCU_Comm_Free(void) -{ - return PCU_Comm_Free_2(&global_pcu); +int PCU_Comm_Free(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Free called before Comm_Init"); + delete global_pcu; + global_pcu = nullptr; + return PCU_SUCCESS; } /** \brief Returns the communication rank of the calling thread. @@ -74,18 +69,20 @@ int PCU_Comm_Free(void) Ranks are contiguous within a process, so that the \f$t\f$ threads in process \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. */ -int PCU_Comm_Self(void) -{ - return PCU_Comm_Self_2(&global_pcu); +int PCU_Comm_Self(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Self called before Comm_Init"); + return global_pcu->Self(); } /** \brief Returns the number of threads in the program. \details when called from a non-threaded MPI process, this function is equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). */ -int PCU_Comm_Peers(void) -{ - return PCU_Comm_Peers_2(&global_pcu); +int PCU_Comm_Peers(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Peers called before Comm_Init"); + return global_pcu->Peers(); } /** \brief Begins a PCU communication phase. @@ -94,9 +91,10 @@ int PCU_Comm_Peers(void) After calling this function, each thread may call functions like PCU_Comm_Pack or PCU_Comm_Write. */ -void PCU_Comm_Begin(void) -{ - PCU_Comm_Begin_2(&global_pcu); +void PCU_Comm_Begin(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Begin called before Comm_Init"); + global_pcu->Begin(); } /** \brief Packs data to be sent to \a to_rank. @@ -105,9 +103,10 @@ void PCU_Comm_Begin(void) This function should be called after PCU_Comm_Start and before PCU_Comm_Send. */ -int PCU_Comm_Pack(int to_rank, const void* data, size_t size) -{ - return PCU_Comm_Pack_2(&global_pcu, to_rank, data, size); +int PCU_Comm_Pack(int to_rank, const void *data, size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Pack called before Comm_Init"); + return global_pcu->Pack(to_rank, data, size); } /** \brief Sends all buffers for this communication phase. @@ -117,9 +116,10 @@ int PCU_Comm_Pack(int to_rank, const void* data, size_t size) All buffers from this thread are sent out and receiving may begin after this call. */ -int PCU_Comm_Send(void) -{ - return PCU_Comm_Send_2(&global_pcu); +int PCU_Comm_Send(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Send called before Comm_Init"); + return global_pcu->Send(); } /** \brief Tries to receive a buffer for this communication phase. @@ -133,25 +133,28 @@ int PCU_Comm_Send(void) Users should unpack all data from this buffer before calling this function again, because the previously received buffer is destroyed by the call. */ -bool PCU_Comm_Listen(void) -{ - return PCU_Comm_Listen_2(&global_pcu); +bool PCU_Comm_Listen(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Listen called before Comm_Init"); + return global_pcu->Listen(); } /** \brief Returns in * \a from_rank the sender of the current received buffer. \details This function should be called after a successful PCU_Comm_Listen. */ -int PCU_Comm_Sender(void) -{ - return PCU_Comm_Sender_2(&global_pcu); +int PCU_Comm_Sender(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Sender called before Comm_Init"); + return global_pcu->Sender(); } /** \brief Returns true if the current received buffer has been unpacked. \details This function should be called after a successful PCU_Comm_Listen. */ -bool PCU_Comm_Unpacked(void) -{ - return PCU_Comm_Unpacked_2(&global_pcu); +bool PCU_Comm_Unpacked(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Unpacked called before Comm_Init"); + return global_pcu->Unpacked(); } /** \brief Unpacks a block of data from the current received buffer. @@ -164,20 +167,23 @@ bool PCU_Comm_Unpacked(void) Users must ensure that there remain \a size bytes to be unpacked, PCU_Comm_Unpacked can help with this. */ -int PCU_Comm_Unpack(void* data, size_t size) -{ - return PCU_Comm_Unpack_2(&global_pcu, data, size); +int PCU_Comm_Unpack(void *data, size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Unpack called before Comm_Init"); + return global_pcu->Unpack(data, size); } -void PCU_Comm_Order(bool on) -{ - PCU_Comm_Order_2(&global_pcu, on); +void PCU_Comm_Order(bool on) { + if (global_pcu == nullptr) + reel_fail("Comm_Order called before Comm_Init"); + global_pcu->Order(on); } /** \brief Blocking barrier over all threads. */ -void PCU_Barrier(void) -{ - PCU_Barrier_2(&global_pcu); +void PCU_Barrier(void) { + if (global_pcu == nullptr) + reel_fail("Barrier called before Comm_Init"); + global_pcu->Barrier(); } /** \brief Performs an Allreduce sum of double arrays. @@ -186,94 +192,114 @@ void PCU_Barrier(void) After this call, p[i] will contain the sum of all p[i]'s given by each rank. */ -void PCU_Add_Doubles(double* p, size_t n) -{ - PCU_Add_Doubles_2(&global_pcu, p, n); +void PCU_Add_Doubles(double *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_Doubles called before Comm_Init"); + global_pcu->Add(p, n); } -double PCU_Add_Double(double x) -{ - return PCU_Add_Double_2(&global_pcu, x); +double PCU_Add_Double(double x) { + if (global_pcu == nullptr) + reel_fail("Add_Double called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce minimum of double arrays. - */ -void PCU_Min_Doubles(double* p, size_t n) -{ - PCU_Min_Doubles_2(&global_pcu, p, n); + */ +void PCU_Min_Doubles(double *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Min_Doubles called before Comm_Init"); + global_pcu->Min(p, n); } -double PCU_Min_Double(double x) -{ - return PCU_Min_Double_2(&global_pcu, x); +double PCU_Min_Double(double x) { + if (global_pcu == nullptr) + reel_fail("Min_Double called before Comm_Init"); + return global_pcu->Min(x); } /** \brief Performs an Allreduce maximum of double arrays. - */ -void PCU_Max_Doubles(double* p, size_t n) -{ - PCU_Max_Doubles_2(&global_pcu, p, n); + */ +void PCU_Max_Doubles(double *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_Doubles called before Comm_Init"); + global_pcu->Max(p, n); } -double PCU_Max_Double(double x) -{ - return PCU_Max_Double_2(&global_pcu, x); +double PCU_Max_Double(double x) { + if (global_pcu == nullptr) + reel_fail("Max_Double called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs an Allreduce sum of integers - */ -void PCU_Add_Ints(int* p, size_t n) -{ - PCU_Add_Ints_2(&global_pcu, p, n); + */ +void PCU_Add_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_Ints called before Comm_Init"); + global_pcu->Add(p, n); } -int PCU_Add_Int(int x) -{ - return PCU_Add_Int_2(&global_pcu, x); +int PCU_Add_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Add_Int called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce sum of long integers - */ -void PCU_Add_Longs(long* p, size_t n) -{ - PCU_Add_Longs_2(&global_pcu, p, n); + */ +void PCU_Add_Longs(long *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_Longs called before Comm_Init"); + global_pcu->Add(p, n); } -long PCU_Add_Long(long x) -{ - return PCU_Add_Long_2(&global_pcu, x); +long PCU_Add_Long(long x) { + if (global_pcu == nullptr) + reel_fail("Add_Long called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce sum of size_t unsigned integers - */ -void PCU_Add_SizeTs(size_t* p, size_t n) -{ - PCU_Add_SizeTs_2(&global_pcu, p, n); + */ +void PCU_Add_SizeTs(size_t *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Add_SizeTs called before Comm_Init"); + global_pcu->Add(p, n); } -size_t PCU_Add_SizeT(size_t x) -{ - return PCU_Add_SizeT_2(&global_pcu, x); +size_t PCU_Add_SizeT(size_t x) { + if (global_pcu == nullptr) + reel_fail("Add_SizeT called before Comm_Init"); + return global_pcu->Add(x); } /** \brief Performs an Allreduce minimum of size_t unsigned integers - */ -void PCU_Min_SizeTs(size_t* p, size_t n) { - PCU_Min_SizeTs_2(&global_pcu, p, n); + */ +void PCU_Min_SizeTs(size_t *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Min_SizeTs called before Comm_Init"); + global_pcu->Min(p, n); } size_t PCU_Min_SizeT(size_t x) { - return PCU_Min_SizeT_2(&global_pcu, x); + if (global_pcu == nullptr) + reel_fail("Min_SizeT called before Comm_Init"); + return global_pcu->Min(x); } /** \brief Performs an Allreduce maximum of size_t unsigned integers - */ -void PCU_Max_SizeTs(size_t* p, size_t n) { - PCU_Max_SizeTs_2(&global_pcu, p, n); + */ +void PCU_Max_SizeTs(size_t *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_SizeTs called before Comm_Init"); + global_pcu->Max(p, n); } size_t PCU_Max_SizeT(size_t x) { - return PCU_Max_SizeT_2(&global_pcu, x); + if (global_pcu == nullptr) + reel_fail("Max_SizeT called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs an exclusive prefix sum of integer arrays. @@ -282,115 +308,129 @@ size_t PCU_Max_SizeT(size_t x) { After this call, p[i] will contain the sum of all p[i]'s given by ranks lower than the calling rank. */ -void PCU_Exscan_Ints(int* p, size_t n) -{ - PCU_Exscan_Ints_2(&global_pcu, p, n); +void PCU_Exscan_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Exscan_Ints called before Comm_Init"); + global_pcu->Exscan(p, n); } -int PCU_Exscan_Int(int x) -{ - return PCU_Exscan_Int_2(&global_pcu, x); +int PCU_Exscan_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Exscan_Int called before Comm_Init"); + return global_pcu->Exscan(x); } /** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs(long* p, size_t n) -{ - PCU_Exscan_Longs_2(&global_pcu, p, n); +void PCU_Exscan_Longs(long *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Exscan_Longs called before Comm_Init"); + global_pcu->Exscan(p, n); } -long PCU_Exscan_Long(long x) -{ - return PCU_Exscan_Long_2(&global_pcu, x); +long PCU_Exscan_Long(long x) { + if (global_pcu == nullptr) + reel_fail("Exscan_Long called before Comm_Init"); + return global_pcu->Exscan(x); } /** \brief Performs an Allreduce minimum of int arrays. - */ -void PCU_Min_Ints(int* p, size_t n) -{ - PCU_Min_Ints_2(&global_pcu, p, n); + */ +void PCU_Min_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Min_Ints called before Comm_Init"); + global_pcu->Min(p, n); } -int PCU_Min_Int(int x) -{ - return PCU_Min_Int_2(&global_pcu, x); +int PCU_Min_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Min_Int called before Comm_Init"); + return global_pcu->Min(x); } /** \brief Performs an Allreduce maximum of int arrays. - */ -void PCU_Max_Ints(int* p, size_t n) -{ - PCU_Max_Ints_2(&global_pcu, p, n); + */ +void PCU_Max_Ints(int *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_Ints called before Comm_Init"); + global_pcu->Max(p, n); } -int PCU_Max_Int(int x) -{ - return PCU_Max_Int_2(&global_pcu, x); +int PCU_Max_Int(int x) { + if (global_pcu == nullptr) + reel_fail("Max_Int called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs an Allreduce maximum of long arrays. - */ -void PCU_Max_Longs(long* p, size_t n) -{ - PCU_Max_Longs_2(&global_pcu, p, n); + */ +void PCU_Max_Longs(long *p, size_t n) { + if (global_pcu == nullptr) + reel_fail("Max_Longs called before Comm_Init"); + global_pcu->Max(p, n); } -long PCU_Max_Long(long x) -{ - return PCU_Max_Long_2(&global_pcu, x); +long PCU_Max_Long(long x) { + if (global_pcu == nullptr) + reel_fail("Max_Long called before Comm_Init"); + return global_pcu->Max(x); } /** \brief Performs a parallel logical OR reduction - */ -int PCU_Or(int c) -{ - return PCU_Or_2(&global_pcu, c); + */ +int PCU_Or(int c) { + if (global_pcu == nullptr) + reel_fail("Or called before Comm_Init"); + return global_pcu->Or(c); } /** \brief Performs a parallel logical AND reduction - */ -int PCU_And(int c) -{ - return PCU_And_2(&global_pcu, c); + */ +int PCU_And(int c) { + if (global_pcu == nullptr) + reel_fail("And called before Comm_Init"); + return global_pcu->And(c); } /** \brief Returns the unique rank of the calling process. */ -int PCU_Proc_Self(void) -{ - return PCU_Proc_Self_2(&global_pcu); +int PCU_Proc_Self(void) { + if (global_pcu == nullptr) + reel_fail("Proc_Self called before Comm_Init"); + return global_pcu->Self(); } /** \brief Returns the number of processes. */ -int PCU_Proc_Peers(void) -{ - return PCU_Proc_Peers_2(&global_pcu); +int PCU_Proc_Peers(void) { + if (global_pcu == nullptr) + reel_fail("Proc_Peers called before Comm_Init"); + return global_pcu->Peers(); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ -int PCU_Comm_Rank(int* rank) -{ - return PCU_Comm_Rank_2(&global_pcu, rank); +int PCU_Comm_Rank(int *rank) { + if (global_pcu == nullptr) + reel_fail("Comm_Rank called before Comm_Init"); + return global_pcu->Self(); } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size(int* size) -{ - return PCU_Comm_Size_2(&global_pcu, size); +int PCU_Comm_Size(int *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Size called before Comm_Init"); + *size = global_pcu->Peers(); + return PCU_SUCCESS; } /** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized(void) -{ - return PCU_Comm_Initialized_2(&global_pcu); -} +bool PCU_Comm_Initialized(void) { return global_pcu != nullptr; } /** \brief Deprecated, see PCU_Comm_Begin. */ -int PCU_Comm_Start(PCU_Method method) -{ - (void)method; //warning silencer - return PCU_Comm_Start_2(&global_pcu); +int PCU_Comm_Start(PCU_Method method) { + (void)method; // warning silencer + global_pcu->Begin(); + return PCU_SUCCESS; } /** \brief Returns in * \a size the number of bytes being sent to \a to_rank. @@ -398,9 +438,10 @@ int PCU_Comm_Start(PCU_Method method) This function should be called after PCU_Comm_Start and before PCU_Comm_Send. */ -int PCU_Comm_Packed(int to_rank, size_t* size) -{ - return PCU_Comm_Packed_2(&global_pcu, to_rank, size); +int PCU_Comm_Packed(int to_rank, size_t *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Packed called before Comm_Init"); + return global_pcu->Packed(to_rank, size); } /** \brief Packs a message to be sent to \a to_rank. @@ -412,15 +453,17 @@ int PCU_Comm_Packed(int to_rank, size_t* size) PCU_Comm_Send. If this function is used, PCU_Comm_Pack should not be used. */ -int PCU_Comm_Write(int to_rank, const void* data, size_t size) -{ - return PCU_Comm_Write_2(&global_pcu, to_rank, data, size); +int PCU_Comm_Write(int to_rank, const void *data, size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Write called before Comm_Init"); + return global_pcu->Pack(to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive(void) -{ - return PCU_Comm_Receive_2(&global_pcu); +bool PCU_Comm_Receive(void) { + if (global_pcu == nullptr) + reel_fail("Comm_Receive called before Comm_Init"); + return global_pcu->Receive(); } /** \brief Receives a message for this communication phase. @@ -435,32 +478,33 @@ bool PCU_Comm_Receive(void) Note that the address * \a data points into a PCU buffer, so it is strongly recommended that this data be read and not modified. */ -bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) -{ - return PCU_Comm_Read_2(&global_pcu, from_rank, data, size); +bool PCU_Comm_Read(int *from_rank, void **data, size_t *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Read called before Comm_Init"); + return global_pcu->Read(from_rank, data, size); } - - - -void PCU_Debug_Open(void) -{ - PCU_Debug_Open_2(&global_pcu); +void PCU_Debug_Open(void) { + if (global_pcu == nullptr) + reel_fail("Debug_Open called before Comm_Init"); + global_pcu->DebugOpen(); } /** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print(const char* format, ...) -{ +void PCU_Debug_Print(const char *format, ...) { + if (global_pcu == nullptr) + reel_fail("Debug_Print called before Comm_Init"); va_list arglist; va_start(arglist, format); - PCU_Debug_Printv_2(&global_pcu, format, arglist); + global_pcu->DebugPrint(format, arglist); va_end(arglist); } /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From(int* from_rank) -{ - return PCU_Comm_From_2(&global_pcu, from_rank); +int PCU_Comm_From(int *from_rank) { + if (global_pcu == nullptr) + reel_fail("Comm_From called before Comm_Init"); + return global_pcu->From(from_rank); } /** \brief Returns in * \a size the bytes in the current received buffer @@ -468,9 +512,10 @@ int PCU_Comm_From(int* from_rank) The size returned will be the total received size regardless of how much unpacking has been done. */ -int PCU_Comm_Received(size_t* size) -{ - return PCU_Comm_Received_2(&global_pcu, size); +int PCU_Comm_Received(size_t *size) { + if (global_pcu == nullptr) + reel_fail("Comm_Received called before Comm_Init"); + return global_pcu->Received(size); } /** \brief Extracts a block of data from the current received buffer. @@ -479,9 +524,10 @@ int PCU_Comm_Received(size_t* size) and an internal pointer to that data is returned. The returned pointer must not be freed by the user. */ -void* PCU_Comm_Extract(size_t size) -{ - return PCU_Comm_Extract_2(&global_pcu, size); +void *PCU_Comm_Extract(size_t size) { + if (global_pcu == nullptr) + reel_fail("Comm_Extract called before Comm_Init"); + return global_pcu->Extract(size); } /** \brief Reinitializes PCU with a new MPI communicator. @@ -492,9 +538,13 @@ void* PCU_Comm_Extract(size_t size) in the previous communicator. This is a very heavy weight function and should be used sparingly. */ -void PCU_Switch_Comm(MPI_Comm new_comm) -{ - PCU_Switch_Comm_2(&global_pcu, new_comm); +void PCU_Switch_Comm(MPI_Comm new_comm) { + if (global_pcu == nullptr) + reel_fail("Switch_Comm called before Comm_Init"); + if (new_comm != global_pcu->GetMPIComm()) { + delete global_pcu; + global_pcu = new pcu::PCU(new_comm); + } } /** \brief Return the current MPI communicator @@ -502,19 +552,17 @@ void PCU_Switch_Comm(MPI_Comm new_comm) most recent PCU_Switch_Comm call, or MPI_COMM_WORLD otherwise. */ -MPI_Comm PCU_Get_Comm(void) -{ - return PCU_Get_Comm_2(&global_pcu); +MPI_Comm PCU_Get_Comm(void) { + if (global_pcu == nullptr) + reel_fail("Get_Comm called before Comm_Init"); + return global_pcu->GetMPIComm(); } /** \brief Return the time in seconds since some time in the past */ -double PCU_Time(void) -{ - return PCU_Time_2(&global_pcu); -} +double PCU_Time(void) { return pcu::Time(); } + +void PCU_Protect(void) { return pcu::Protect(); } -void PCU_Protect(void) -{ - PCU_Protect_2(&global_pcu); +double PCU_GetMem(void) { return pcu::GetMem(); } } diff --git a/pcu/pcu2.c b/pcu/pcu2.c deleted file mode 100644 index 03b4db62d..000000000 --- a/pcu/pcu2.c +++ /dev/null @@ -1,787 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -/** \file pcu->c - \brief The PCU communication interface */ -/** \page pcu PCU - PCU (the Parallel Control Utility) is a library for parallel computation - based on MPI. - PCU provides three things to users: - 1. A hybrid phased message passing system - 2. Hybrid collective operations - - Phased message passing is similar to Bulk Synchronous Parallel. - All messages are exchanged in a phase, which is a collective operation - involving all threads in the parallel program. - During a phase, the following events happen in sequence: - 1. All threads send non-blocking messages to other threads - 2. All threads receive all messages sent to them during this phase - PCU provides termination detection, which is the ability to detect when all - messages have been received without prior knowledge of which threads - are sending to which. - - The API documentation is here: pcu->c -*/ - -#include -#include -#include "PCU2.h" -#include "pcu_msg.h" -#include "pcu_order.h" -#include "noto_malloc.h" -#include "reel.h" -#include /*required for mode_t for mkdir on some systems*/ -#include /*using POSIX mkdir call for SMB "foo/" path*/ -#include /* for checking the error from mkdir */ -#include /*INT_MAX*/ -#include /*abort*/ - - -static pcu_msg* get_msg(pcu_t * pcu) -{ - return &(pcu->pmsg); -} - -/** \brief Initializes the PCU library. - \details This function must be called by all MPI processes before - calling any other PCU functions. - MPI_Init or MPI_Init_thread should be called before this function. - */ -int PCU_Comm_Init_2(pcu_t* pcu) -{ - if (pcu->state != pcu_state_uninit) - reel_fail("nested calls to Comm_Init"); - pcu_mpi_init(MPI_COMM_WORLD,&(pcu->mpi)); - pcu_make_msg(&(pcu->pmsg)); - pcu->state = pcu_state_init; - /* turn ordering on by default, call - PCU_Comm_Order(false) after PCU_Comm_Init - to disable this */ - PCU_Comm_Order_2(pcu, true); - return PCU_SUCCESS; -} - -/** \brief Frees all PCU library structures. - \details This function must be called by all MPI processes after all - other calls to PCU, and before calling MPI_Finalize. - */ -int PCU_Comm_Free_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Free called before Comm_Init"); - if (pcu->pmsg.order) - pcu_order_free(pcu->pmsg.order); - pcu_free_msg(&(pcu->pmsg)); - pcu_mpi_finalize(&(pcu->mpi)); - pcu->state = pcu_state_uninit; - return PCU_SUCCESS; -} - -/** \brief Returns the communication rank of the calling thread. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_rank(MPI_COMM_WORLD,rank). - - Ranks are consecutive from 0 to \f$pt-1\f$ for a program with - \f$p\f$ processes and \f$t\f$ threads per process. - Ranks are contiguous within a process, so that the \f$t\f$ threads in process - \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. - */ -int PCU_Comm_Self_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(&(pcu->mpi)); -} - -/** \brief Returns the number of threads in the program. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). - */ -int PCU_Comm_Peers_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(&(pcu->mpi)); -} - -/** \brief Begins a PCU communication phase. - \details This function must be called by all threads in the MPI job - at the beginning of each phase of communication. - After calling this function, each thread may call functions like - PCU_Comm_Pack or PCU_Comm_Write. -*/ -void PCU_Comm_Begin_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(&(pcu->mpi), get_msg(pcu)); -} - -/** \brief Packs data to be sent to \a to_rank. - \details This function appends the block of \a size bytes starting - at \a data to the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Pack_2(pcu_t* pcu, int to_rank, const void* data, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) - reel_fail("Invalid rank in Comm_Pack"); - if ( size > (size_t)INT_MAX ) { - fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); - abort(); - } - memcpy(pcu_msg_pack(get_msg(pcu),to_rank,size),data,size); - return PCU_SUCCESS; -} - -/** \brief Sends all buffers for this communication phase. - \details This function should be called by all threads in the MPI job - after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls - to PCU_Comm_Listen or PCU_Comm_Read. - All buffers from this thread are sent out and receiving - may begin after this call. - */ -int PCU_Comm_Send_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(&(pcu->mpi), get_msg(pcu)); - return PCU_SUCCESS; -} - -/** \brief Tries to receive a buffer for this communication phase. - \details Either this function or PCU_Comm_Read should be called at least - once by all threads during the communication phase, after PCU_Comm_Send - is called. The result will be false if and only if the communication phase - is over and there are no more buffers to receive. - Otherwise, a buffer was received. - Its contents are retrievable through PCU_Comm_Unpack, and its metadata through - PCU_Comm_Sender and PCU_Comm_Received. - Users should unpack all data from this buffer before calling this function - again, because the previously received buffer is destroyed by the call. - */ -bool PCU_Comm_Listen_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Listen called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_receive(&(pcu->mpi), m->order, m); - return pcu_msg_receive(&(pcu->mpi), m); -} - -/** \brief Returns in * \a from_rank the sender of the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - */ -int PCU_Comm_Sender_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Sender called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_received_from(m->order); - return pcu_msg_received_from(m); -} - -/** \brief Returns true if the current received buffer has been unpacked. - \details This function should be called after a successful PCU_Comm_Listen. - */ -bool PCU_Comm_Unpacked_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Unpacked called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_unpacked(m->order); - return pcu_msg_unpacked(m); -} - -/** \brief Unpacks a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - \a data must point to a block of memory of at least \a size bytes, into - which the next \a size bytes of the current received buffer will be written. - Subsequent calls will begin unpacking where this call left off, - so that the entire received buffer can be unpacked by a sequence of calls to - this function. - Users must ensure that there remain \a size bytes to be unpacked, - PCU_Comm_Unpacked can help with this. - */ -int PCU_Comm_Unpack_2(pcu_t* pcu, void* data, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Unpack called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - memcpy(data,pcu_order_unpack(m->order,size),size); - else - memcpy(data,pcu_msg_unpack(m,size),size); - return PCU_SUCCESS; -} - -void PCU_Comm_Order_2(pcu_t* pcu, bool on) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Order called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (on && (!m->order)) - m->order = pcu_order_new(); - if ((!on) && m->order) { - pcu_order_free(m->order); - m->order = NULL; - } -} - -/** \brief Blocking barrier over all threads. */ -void PCU_Barrier_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&(pcu->mpi), &(get_msg(pcu)->coll)); -} - -/** \brief Performs an Allreduce sum of double arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n doubles. - After this call, p[i] will contain the sum of all p[i]'s - given by each rank. - */ -void PCU_Add_Doubles_2(pcu_t* pcu, double* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_doubles,p,n*sizeof(double)); -} - -double PCU_Add_Double_2(pcu_t* pcu, double x) -{ - double a[1]; - a[0] = x; - PCU_Add_Doubles_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of double arrays. - */ -void PCU_Min_Doubles_2(pcu_t* pcu, double* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_doubles,p,n*sizeof(double)); -} - -double PCU_Min_Double_2(pcu_t* pcu, double x) -{ - double a[1]; - a[0] = x; - PCU_Min_Doubles_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of double arrays. - */ -void PCU_Max_Doubles_2(pcu_t* pcu, double* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_doubles,p,n*sizeof(double)); -} - -double PCU_Max_Double_2(pcu_t* pcu, double x) -{ - double a[1]; - a[0] = x; - PCU_Max_Doubles_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of integers - */ -void PCU_Add_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); -} - -int PCU_Add_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Add_Ints_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of long integers - */ -void PCU_Add_Longs_2(pcu_t* pcu, long* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); -} - -long PCU_Add_Long_2(pcu_t* pcu, long x) -{ - long a[1]; - a[0] = x; - PCU_Add_Longs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of size_t unsigned integers - */ -void PCU_Add_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Add_SizeT_2(pcu_t* pcu, size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Add_SizeTs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of size_t unsigned integers - */ -void PCU_Min_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Min_SizeT_2(pcu_t* pcu, size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Min_SizeTs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of size_t unsigned integers - */ -void PCU_Max_SizeTs_2(pcu_t* pcu, size_t* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Max_SizeT_2(pcu_t* pcu, size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Max_SizeTs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an exclusive prefix sum of integer arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n integers. - After this call, p[i] will contain the sum of all p[i]'s - given by ranks lower than the calling rank. - */ -void PCU_Exscan_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Exscan_Ints called before Comm_Init"); - int* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_ints,p,n*sizeof(int)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); -} - -int PCU_Exscan_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Exscan_Ints_2(pcu, a, 1); - return a[0]; -} - -/** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs_2(pcu_t* pcu, long* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Exscan_Longs called before Comm_Init"); - long* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_add_longs,p,n*sizeof(long)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); -} - -long PCU_Exscan_Long_2(pcu_t* pcu, long x) -{ - long a[1]; - a[0] = x; - PCU_Exscan_Longs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of int arrays. - */ -void PCU_Min_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_min_ints,p,n*sizeof(int)); -} - -int PCU_Min_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Min_Ints_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of int arrays. - */ -void PCU_Max_Ints_2(pcu_t* pcu, int* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_ints,p,n*sizeof(int)); -} - -int PCU_Max_Int_2(pcu_t* pcu, int x) -{ - int a[1]; - a[0] = x; - PCU_Max_Ints_2(pcu, a, 1); - return a[0]; -} -/** \brief Performs an Allreduce maximum of long arrays. - */ -void PCU_Max_Longs_2(pcu_t* pcu, long* p, size_t n) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&(pcu->mpi), &(get_msg(pcu)->coll),pcu_max_longs,p,n*sizeof(long)); -} - -long PCU_Max_Long_2(pcu_t* pcu, long x) -{ - long a[1]; - a[0] = x; - PCU_Max_Longs_2(pcu, a, 1); - return a[0]; -} - -/** \brief Performs a parallel logical OR reduction - */ -int PCU_Or_2(pcu_t* pcu, int c) -{ - return PCU_Max_Int_2(pcu, c); -} - -/** \brief Performs a parallel logical AND reduction - */ -int PCU_And_2(pcu_t* pcu, int c) -{ - return PCU_Min_Int_2(pcu, c); -} - -/** \brief Returns the unique rank of the calling process. - */ -int PCU_Proc_Self_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Proc_Self called before Comm_Init"); - return pcu_mpi_rank(&(pcu->mpi)); -} - -/** \brief Returns the number of processes. - */ -int PCU_Proc_Peers_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Proc_Peers called before Comm_Init"); - return pcu_mpi_size(&(pcu->mpi)); -} - -/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. - */ -int PCU_Comm_Rank_2(pcu_t* pcu, int* rank) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(&(pcu->mpi)); - return PCU_SUCCESS; -} - -/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size_2(pcu_t* pcu, int* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(&(pcu->mpi)); - return PCU_SUCCESS; -} - -/** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized_2(pcu_t* pcu) -{ - return pcu->state == pcu_state_init; -} - -/** \brief Deprecated, see PCU_Comm_Begin. - */ -int PCU_Comm_Start_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(&(pcu->mpi),get_msg(pcu)); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. - \details Returns the size of the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Packed_2(pcu_t* pcu, int to_rank, size_t* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) - reel_fail("Invalid rank in Comm_Packed"); - *size = pcu_msg_packed(get_msg(pcu),to_rank); - return PCU_SUCCESS; -} - -/** \brief Packs a message to be sent to \a to_rank. - \details This function packs a message into the buffer being sent - to \a to_rank. - Messages packed by this function can be received using the function - PCU_Comm_Read. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - If this function is used, PCU_Comm_Pack should not be used. - */ -int PCU_Comm_Write_2(pcu_t* pcu, int to_rank, const void* data, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size(&(pcu->mpi)))) - reel_fail("Invalid rank in Comm_Write"); - pcu_msg* msg = get_msg(pcu); - PCU_MSG_PACK(msg,to_rank,size); - memcpy(pcu_msg_pack(msg,to_rank,size),data,size); - return PCU_SUCCESS; -} - -/** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive_2(pcu_t* pcu) -{ - while (PCU_Comm_Unpacked_2(pcu)) - if (!PCU_Comm_Listen_2(pcu)) - return false; - return true; -} - -/** \brief Receives a message for this communication phase. - \details This function tries to receive a message packed by - PCU_Comm_Write. - If a the communication phase is over and there are no more - messages to receive, this function returns false. - Otherwise, * \a from_rank will be the rank which sent the message, - * \a data will point to the start of the message data, and - * \a size will be the number of bytes of message data. - If this function is used, PCU_Comm_Receive should not be used. - Note that the address * \a data points into a PCU buffer, so - it is strongly recommended that this data be read and not modified. - */ -bool PCU_Comm_Read_2(pcu_t* pcu, int* from_rank, void** data, size_t* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Read called before Comm_Init"); - if (!PCU_Comm_Receive_2(pcu)) - return false; - *from_rank = PCU_Comm_Sender_2(pcu); - PCU_Comm_Unpack_2(pcu, size, sizeof(*size)); - *data = PCU_Comm_Extract_2(pcu, *size); - return true; -} - -static void safe_mkdir(const char* path, mode_t mode) -{ - int err; - errno = 0; - err = mkdir(path, mode); - if (err != 0 && errno != EEXIST) - reel_fail("PCU: could not create directory \"%s\"\n", path); -} - -static void append(char* s, size_t size, const char* format, ...) -{ - int len = strlen(s); - va_list ap; - va_start(ap, format); - vsnprintf(s + len, size - len, format, ap); - va_end(ap); -} - - -void PCU_Debug_Open_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Debug_Open called before Comm_Init"); - - const int fanout = 2048; - const int bufsize = 1024; - char* path = noto_malloc(bufsize); - path[0] = '\0'; - if (PCU_Comm_Peers_2(pcu) > fanout) { - mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - strcpy(path, "debug/"); - safe_mkdir(path, dir_perm); - int self = PCU_Comm_Self_2(pcu); - append(path, bufsize, "%d/", self / fanout); - if (self % fanout == 0) - safe_mkdir(path, dir_perm); - PCU_Barrier_2(pcu); - } - - append(path,bufsize, "%s", "debug"); - pcu_msg* msg = get_msg(pcu); - if ( ! msg->file) - msg->file = pcu_open_parallel(path,"txt"); - noto_free(path); -} - -void PCU_Debug_Printv_2(pcu_t* pcu, const char* format, va_list ap) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Debug_Print called before Comm_Init"); - pcu_msg* msg = get_msg(pcu); - if ( ! msg->file) - return; //Print is a no-op if no file is open - vfprintf(msg->file,format,ap); - fflush(msg->file); -} -/** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print_2(pcu_t* pcu, const char* format, ...) -{ - va_list arglist; - va_start(arglist,format); - PCU_Debug_Printv_2(pcu, format, arglist); - va_end(arglist); -} - -/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From_2(pcu_t* pcu, int* from_rank) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_From called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - *from_rank = pcu_order_received_from(m->order); - else - *from_rank = pcu_msg_received_from(m); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the bytes in the current received buffer - \details This function should be called after a successful PCU_Comm_Receive. - The size returned will be the total received size regardless of how - much unpacking has been done. - */ -int PCU_Comm_Received_2(pcu_t* pcu, size_t* size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Received called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - *size = pcu_order_received_size(m->order); - else - *size = pcu_msg_received_size(m); - return PCU_SUCCESS; -} - -/** \brief Extracts a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Receive. - The next \a size bytes of the current received buffer are unpacked, - and an internal pointer to that data is returned. - The returned pointer must not be freed by the user. - */ -void* PCU_Comm_Extract_2(pcu_t* pcu, size_t size) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Comm_Extract called before Comm_Init"); - pcu_msg* m = get_msg(pcu); - if (m->order) - return pcu_order_unpack(m->order,size); - return pcu_msg_unpack(m,size); -} - -/** \brief Reinitializes PCU with a new MPI communicator. - \details All of PCU's logic is based off two duplicates - of this communicator, so you can safely get PCU to act - on sub-groups of processes using this function. - This call should be collective over all processes - in the previous communicator. This is a very heavy weight function - and should be used sparingly. - */ -void PCU_Switch_Comm_2(pcu_t* pcu, MPI_Comm new_comm) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Switch_Comm called before Comm_Init"); - int result; - MPI_Comm_compare(new_comm, pcu->mpi.original_comm, &result); - if (result != MPI_IDENT) { - pcu_mpi_finalize(&(pcu->mpi)); - pcu_mpi_init(new_comm,&(pcu->mpi)); - } -} - -/** \brief Return the current MPI communicator - \details Returns the communicator given to the - most recent PCU_Switch_Comm call, or MPI_COMM_WORLD - otherwise. - */ -MPI_Comm PCU_Get_Comm_2(pcu_t* pcu) -{ - if (pcu->state == pcu_state_uninit) - reel_fail("Get_Comm called before Comm_Init"); - return pcu->mpi.original_comm; -} - -/** \brief Return the time in seconds since some time in the past - */ -double PCU_Time_2(pcu_t* pcu) -{ - // silence warning - (void)pcu; - return MPI_Wtime(); -} - -void PCU_Protect_2(pcu_t* pcu) -{ - // silence warning - (void)pcu; - reel_protect(); -} diff --git a/pcu/pcu_buffer.h b/pcu/pcu_buffer.h index e8ff30d22..7695eacee 100644 --- a/pcu/pcu_buffer.h +++ b/pcu/pcu_buffer.h @@ -12,6 +12,9 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif typedef struct { @@ -28,5 +31,8 @@ void* pcu_walk_buffer(pcu_buffer* b, size_t size); bool pcu_buffer_walked(pcu_buffer* b); void pcu_resize_buffer(pcu_buffer* b, size_t size); void pcu_set_buffer(pcu_buffer* b, void* p, size_t size); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_byteorder.h b/pcu/pcu_byteorder.h index 6621b01cb..7ca91c9dd 100644 --- a/pcu/pcu_byteorder.h +++ b/pcu/pcu_byteorder.h @@ -6,6 +6,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + #if CHAR_BIT != 8 #error "unsupported char size" #endif @@ -22,4 +26,8 @@ static const union { #define PCU_HOST_ORDER (pcu_host_order.value) +#ifdef __cplusplus +} +#endif + #endif diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index a3ad4e497..af888c048 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -33,6 +33,7 @@ void pcu_merge_assign(void* local, void* incoming, size_t size) memcpy(local,incoming,size); } +// TODO remove once struct fully inplace void pcu_add_doubles(void* local, void* incoming, size_t size) { double* a = local; diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index 9c89d64f9..aaaf5a6ff 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -12,6 +12,10 @@ #include "pcu_mpi.h" +#ifdef __cplusplus +extern "C" { +#endif + /* The PCU Collectives system (pcu_coll for short) implements non-blocking collective operations based loosely on binary-tree or binomial communication patterns. @@ -88,4 +92,7 @@ void pcu_begin_barrier(pcu_mpi_t*,pcu_coll* c); bool pcu_barrier_done(pcu_mpi_t*, pcu_coll* c); void pcu_barrier(pcu_mpi_t*, pcu_coll* c); +#ifdef __cplusplus +} +#endif #endif //PCU_COLL_H diff --git a/pcu/pcu_defines.h b/pcu/pcu_defines.h new file mode 100644 index 000000000..c39152d16 --- /dev/null +++ b/pcu/pcu_defines.h @@ -0,0 +1,13 @@ +#ifndef SCOREC_PCU_PCU_DEFINES_H +#define SCOREC_PCU_PCU_DEFINES_H + +#define PCU_SUCCESS 0 +#define PCU_FAILURE -1 +#ifdef __GNUC__ +#define PCU_FORMAT_ATTRIBUTE(...) \ + __attribute__((format(printf, ##__VA_ARGS__))); +#else +#define PCU_FORMAT_ATTRIBUTE(format, ...) +#endif + +#endif // SCOREC_PCU_PCU_DEFINES_H diff --git a/pcu/pcu_mem.c b/pcu/pcu_mem.c index 8ac2345c1..bf060b104 100644 --- a/pcu/pcu_mem.c +++ b/pcu/pcu_mem.c @@ -7,7 +7,16 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include +#include +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif #if defined(__APPLE__) @@ -35,7 +44,7 @@ #endif -double PCU_GetMem() { +double pcu_get_mem() { const double M = 1024*1024; #if defined(__APPLE__) bool resident = true; diff --git a/pcu/pcu_mem.h b/pcu/pcu_mem.h new file mode 100644 index 000000000..50f7419b6 --- /dev/null +++ b/pcu/pcu_mem.h @@ -0,0 +1,13 @@ +#ifndef SCOREC_PCU_PCU_MEM_H +#define SCOREC_PCU_PCU_MEM_H + +#ifdef __cplusplus +extern "C" { +#endif + +double pcu_get_mem(); + +#ifdef __cplusplus +} +#endif +#endif // SCOREC_PCU_PCU_MEM_H diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index fb55ea275..970c8a4f2 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -13,6 +13,10 @@ #include "pcu_buffer.h" #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { pcu_buffer buffer; @@ -23,14 +27,15 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); -typedef struct +struct pcu_mpi_struct { MPI_Comm original_comm; MPI_Comm user_comm; MPI_Comm coll_comm; int rank; int size; -} pcu_mpi_t; +}; +typedef struct pcu_mpi_struct pcu_mpi_t; int pcu_mpi_size(const pcu_mpi_t*); int pcu_mpi_rank(const pcu_mpi_t*); @@ -40,4 +45,7 @@ bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); void pcu_mpi_finalize(pcu_mpi_t* mpi); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_msg.h b/pcu/pcu_msg.h index 0289d0d16..ba7595ff6 100644 --- a/pcu/pcu_msg.h +++ b/pcu/pcu_msg.h @@ -14,6 +14,10 @@ #include "pcu_aa.h" #include "pcu_io.h" +#ifdef __cplusplus +extern "C" { +#endif + /* the PCU Messenger (pcu_msg for short) system implements a non-blocking Bulk Synchronous Parallel communication model it is based on PCU non-blocking Collectives and pcu_mpi, @@ -60,4 +64,7 @@ int pcu_msg_received_from(pcu_msg* m); size_t pcu_msg_received_size(pcu_msg* m); void pcu_free_msg(pcu_msg* m); +#ifdef __cplusplus +} +#endif #endif //PCU_MSG_H diff --git a/pcu/pcu_order.h b/pcu/pcu_order.h index 26e144885..1aa870edc 100644 --- a/pcu/pcu_order.h +++ b/pcu/pcu_order.h @@ -13,6 +13,9 @@ #include #include "pcu_msg.h" +#ifdef __cplusplus +extern "C" { +#endif typedef struct pcu_order_struct* pcu_order; pcu_order pcu_order_new(void); @@ -23,4 +26,7 @@ bool pcu_order_unpacked(pcu_order o); int pcu_order_received_from(pcu_order o); size_t pcu_order_received_size(pcu_order o); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index d6dd08dae..7505aa7a4 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2011 Scientific Computation Research Center, + Copyright 2011 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -13,13 +13,18 @@ #include "pcu_mpi.h" #include +#ifdef __cplusplus +extern "C" { +#endif +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t *mpi); +void pcu_pmpi_finalize(pcu_mpi_t *m); +int pcu_pmpi_size(const pcu_mpi_t *self); +int pcu_pmpi_rank(const pcu_mpi_t *self); +void pcu_pmpi_send(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +bool pcu_pmpi_receive(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +bool pcu_pmpi_done(const pcu_mpi_t *, pcu_message *m); -void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* mpi); -void pcu_pmpi_finalize(pcu_mpi_t* m); -int pcu_pmpi_size(const pcu_mpi_t* self); -int pcu_pmpi_rank(const pcu_mpi_t* self); -void pcu_pmpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_done(const pcu_mpi_t*, pcu_message* m); - +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_util.h b/pcu/pcu_util.h index 63aa49a30..e4a248408 100644 --- a/pcu/pcu_util.h +++ b/pcu/pcu_util.h @@ -22,17 +22,32 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); } /* extern "C" */ #endif -#define PCU_ALWAYS_ASSERT(cond) \ - do { \ - if (! (cond)) { \ - char omsg[2048]; \ - sprintf(omsg, "%s failed at %s + %d \n", \ - #cond, __FILE__, __LINE__); \ - PCU_Assert_Fail(omsg); \ - } \ - } while (0) +#define PCU_DO_PRAGMA_(x) _Pragma (#x) +#define PCU_DO_PRAGMA(x) PCU_DO_PRAGMA_(x) + +#define PCU_IGNORE_DIAGNOSTIC_START(x) \ + PCU_DO_PRAGMA(GCC diagnostic push) \ + PCU_DO_PRAGMA(GCC diagnostic ignored #x) \ + PCU_DO_PRAGMA(clang diagnostic push) \ + PCU_DO_PRAGMA(clang diagnostic ignored #x) + +#define PCU_IGNORE_DIAGNOSTIC_END \ + PCU_DO_PRAGMA(GCC diagnostic pop) \ + PCU_DO_PRAGMA(clang diagnostic pop) +#define PCU_ALWAYS_ASSERT(cond) \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ + do { \ + if (! (cond)) { \ + char omsg[2048]; \ + sprintf(omsg, "%s failed at %s + %d \n", \ + #cond, __FILE__, __LINE__); \ + PCU_Assert_Fail(omsg); \ + } \ + } while (0) \ + PCU_IGNORE_DIAGNOSTIC_END #define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ do { \ if (! (cond)) { \ char omsg[2048]; \ @@ -40,7 +55,8 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); #cond, __FILE__, __LINE__, msg); \ PCU_Assert_Fail(omsg); \ } \ - } while(0) + } while(0) \ + PCU_IGNORE_DIAGNOSTIC_END #ifdef NDEBUG #define PCU_DEBUG_ASSERT(cond) diff --git a/pcu/pkg_tribits.cmake b/pcu/pkg_tribits.cmake index 0c43fc64a..ce6e88e92 100644 --- a/pcu/pkg_tribits.cmake +++ b/pcu/pkg_tribits.cmake @@ -33,27 +33,28 @@ include_directories("${PROJECT_BINARY_DIR}") #Sources & Headers set(SOURCES - pcu.c - pcu_aa.c - pcu_coll.c - pcu_io.c - pcu_buffer.c - pcu_mpi.c - pcu_msg.c - pcu_order.c - pcu_pmpi.c - pcu_util.c - noto/noto_malloc.c - reel/reel.c -) + pcu.cc + pcu_aa.c + pcu_coll.c + pcu_io.c + pcu_buffer.c + pcu_mem.c + pcu_mpi.c + pcu_msg.c + pcu_order.c + pcu_pmpi.c + pcu_util.c + noto/noto_malloc.c + reel/reel.c + PCUObj.cc) set(HEADERS - PCU.h - pcu_io.h - pcu_util.h - noto/noto_malloc.h - reel/reel.h) - + PCU.h + pcu_io.h + pcu_util.h + reel/reel.h + pcu_defines.h + ) tribits_add_library( pcu HEADERS ${HEADERS} From 23c7cefaf0bb1bb807e6ba20bf11bd8f26497034 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 02:14:49 -0400 Subject: [PATCH 026/141] remove unused functions --- pcu/pcu_coll.c | 100 ------------------------------------------------- pcu/pcu_coll.h | 11 ------ 2 files changed, 111 deletions(-) diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index af888c048..01c89445f 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -33,106 +33,6 @@ void pcu_merge_assign(void* local, void* incoming, size_t size) memcpy(local,incoming,size); } -// TODO remove once struct fully inplace -void pcu_add_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_max_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_min_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_add_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_min_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_max_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_max_longs(void* local, void* incoming, size_t size) -{ - long* a = local; - long* b= incoming; - size_t n = size/sizeof(long); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_min_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_max_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_add_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_add_longs(void* local, void* incoming, size_t size) -{ - long* a = local; - long* b= incoming; - size_t n = size/sizeof(long); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - /* initiates non-blocking calls for this communication step */ static void begin_coll_step(pcu_mpi_t* mpi, pcu_coll* c) diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index aaaf5a6ff..f69bf7a08 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -31,17 +31,6 @@ extern "C" { typedef void pcu_merge(void* local, void* incoming, size_t size); void pcu_merge_assign(void* local, void* incoming, size_t size); -void pcu_add_doubles(void* local, void* incoming, size_t size); -void pcu_max_doubles(void* local, void* incoming, size_t size); -void pcu_min_doubles(void* local, void* incoming, size_t size); -void pcu_add_ints(void* local, void* incoming, size_t size); -void pcu_min_ints(void* local, void* incoming, size_t size); -void pcu_max_ints(void* local, void* incoming, size_t size); -void pcu_max_longs(void* local, void* incoming, size_t size); -void pcu_add_longs(void* local, void* incoming, size_t size); -void pcu_add_sizets(void* local, void* incoming, size_t size); -void pcu_min_sizets(void* local, void* incoming, size_t size); -void pcu_max_sizets(void* local, void* incoming, size_t size); /* Enumerated actions that a rank takes during one step of the communication pattern */ From 438a73e8bc8263751d88f20882a2e0660fc7136a Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 02:33:40 -0400 Subject: [PATCH 027/141] Fix gcc compilation --- pcu/PCUObj.cc | 4 ++++ pcu/pcu.cc | 3 ++- pcu/pcu_util.h | 30 +++++++++++++++++++++--------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index a6a3f9b76..d0f6f80bc 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -7,6 +7,10 @@ #include "reel.h" #include #include /*using POSIX mkdir call for SMB "foo/" path*/ +#include +#include +#include +#include namespace pcu { int PCU::Peers() const noexcept { return pcu_mpi_size(mpi_); } diff --git a/pcu/pcu.cc b/pcu/pcu.cc index 3a0337812..aeb1f6da1 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -411,7 +411,8 @@ int PCU_Proc_Peers(void) { int PCU_Comm_Rank(int *rank) { if (global_pcu == nullptr) reel_fail("Comm_Rank called before Comm_Init"); - return global_pcu->Self(); + *rank = global_pcu->Self(); + return PCU_SUCCESS; } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ diff --git a/pcu/pcu_util.h b/pcu/pcu_util.h index e4a248408..8122599ad 100644 --- a/pcu/pcu_util.h +++ b/pcu/pcu_util.h @@ -25,38 +25,50 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); #define PCU_DO_PRAGMA_(x) _Pragma (#x) #define PCU_DO_PRAGMA(x) PCU_DO_PRAGMA_(x) +#if defined(__GNUC__) + #define PCU_IGNORE_DIAGNOSTIC_START(x) \ PCU_DO_PRAGMA(GCC diagnostic push) \ - PCU_DO_PRAGMA(GCC diagnostic ignored #x) \ + PCU_DO_PRAGMA(GCC diagnostic ignored #x) +#define PCU_IGNORE_DIAGNOSTIC_END \ + PCU_DO_PRAGMA(GCC diagnostic pop) + +#elif defined(__clang__) + +#define PCU_IGNORE_DIAGNOSTIC_START(x) \ PCU_DO_PRAGMA(clang diagnostic push) \ PCU_DO_PRAGMA(clang diagnostic ignored #x) - #define PCU_IGNORE_DIAGNOSTIC_END \ - PCU_DO_PRAGMA(GCC diagnostic pop) \ PCU_DO_PRAGMA(clang diagnostic pop) +#else +#define PCU_IGNORE_DIAGNOSTIC_START(x) +#define PCU_IGNORE_DIAGNOSTIC_END +#endif + + #define PCU_ALWAYS_ASSERT(cond) \ - PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ if (! (cond)) { \ char omsg[2048]; \ sprintf(omsg, "%s failed at %s + %d \n", \ #cond, __FILE__, __LINE__); \ PCU_Assert_Fail(omsg); \ } \ - } while (0) \ - PCU_IGNORE_DIAGNOSTIC_END + PCU_IGNORE_DIAGNOSTIC_END \ + } while (0) #define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ - PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations)\ if (! (cond)) { \ char omsg[2048]; \ sprintf(omsg, "%s failed at %s + %d \n %s", \ #cond, __FILE__, __LINE__, msg); \ PCU_Assert_Fail(omsg); \ } \ - } while(0) \ - PCU_IGNORE_DIAGNOSTIC_END + PCU_IGNORE_DIAGNOSTIC_END \ + } while(0) #ifdef NDEBUG #define PCU_DEBUG_ASSERT(cond) From 4fa6e37e9cfe21be77371e6a9384931de6a468d3 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Thu, 30 Mar 2023 20:28:17 -0400 Subject: [PATCH 028/141] fix bug --- pcu/PCUObj.cc | 20 ++++++++++++++++---- pcu/PCUObj.h | 17 ++++++++++++++--- pcu/pcu.cc | 7 ++----- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index d0f6f80bc..40fe39ee5 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -31,7 +31,7 @@ int PCU::Send() noexcept { pcu_msg_send(mpi_, msg_); return PCU_SUCCESS; } -bool PCU::Receive(void) noexcept { +bool PCU::Receive() noexcept { while (Unpacked()) if (!Listen()) return false; @@ -149,9 +149,9 @@ void PCU::DebugOpen() noexcept { noto_free(path); } -double GetMem(void) noexcept { return pcu_get_mem(); } -void Protect(void) noexcept { reel_protect(); } -double Time(void) noexcept { return MPI_Wtime(); } +double GetMem() noexcept { return pcu_get_mem(); } +void Protect() noexcept { reel_protect(); } +double Time() noexcept { return MPI_Wtime(); } void PCU::DebugPrint(const char *format, ...) noexcept { va_list args; @@ -176,7 +176,9 @@ PCU::PCU(MPI_Comm comm) { Order(true); } PCU::~PCU() noexcept { + pcu_mpi_finalize(mpi_); delete mpi_; + pcu_free_msg(msg_); delete msg_; } PCU::PCU(PCU &&other) noexcept { @@ -190,6 +192,16 @@ PCU &PCU::operator=(PCU && other) noexcept { } MPI_Comm PCU::GetMPIComm() const noexcept { return mpi_->original_comm; } +MPI_Comm PCU::SwitchMPIComm(MPI_Comm newcomm) noexcept { + if(newcomm == mpi_->original_comm) { + return mpi_->original_comm; + } + auto original_comm = mpi_->original_comm; + pcu_mpi_finalize(mpi_); + pcu_mpi_init(newcomm, mpi_); + return original_comm; +} + /* template implementations */ template void PCU::Add(T *p, size_t n) noexcept { pcu_allreduce( diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index 9584f5b6c..aea6f263c 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -69,16 +69,27 @@ class PCU { /* Debug functions */ void DebugOpen() noexcept; + MPI_Comm SwitchMPIComm(MPI_Comm) noexcept; + + //struct MPIComms { + // MPI_Comm original; + // MPI_Comm user; + // MPI_Comm coll; + //}; + // takes ownership of newcomms.user & newcomms.coll + // user responsibility to free returned user/coll comm + //MPIComms SwitchMPIComms(MPIComms& newcomms) noexcept; + private: pcu_msg_struct *msg_; pcu_mpi_struct *mpi_; }; /*stack trace helpers using GNU/Linux*/ -void Protect(void) noexcept; +void Protect() noexcept; /*Memory usage*/ -[[nodiscard]] double GetMem(void) noexcept; +[[nodiscard]] double GetMem() noexcept; /*MPI_Wtime() equivalent*/ -[[nodiscard]] double Time(void) noexcept; +[[nodiscard]] double Time() noexcept; /* explicit instantiations of template functions */ #define PCU_EXPL_INST_DECL(T) \ diff --git a/pcu/pcu.cc b/pcu/pcu.cc index aeb1f6da1..0368ff6ff 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -457,7 +457,7 @@ int PCU_Comm_Packed(int to_rank, size_t *size) { int PCU_Comm_Write(int to_rank, const void *data, size_t size) { if (global_pcu == nullptr) reel_fail("Comm_Write called before Comm_Init"); - return global_pcu->Pack(to_rank, data, size); + return global_pcu->Write(to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ @@ -542,10 +542,7 @@ void *PCU_Comm_Extract(size_t size) { void PCU_Switch_Comm(MPI_Comm new_comm) { if (global_pcu == nullptr) reel_fail("Switch_Comm called before Comm_Init"); - if (new_comm != global_pcu->GetMPIComm()) { - delete global_pcu; - global_pcu = new pcu::PCU(new_comm); - } + global_pcu->SwitchMPIComm(new_comm); } /** \brief Return the current MPI communicator From 716133e0a305f86fddd286775e52ec27027d0aef Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Fri, 31 Mar 2023 01:24:21 -0400 Subject: [PATCH 029/141] remove unnecessary whitespace --- pcu/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 3a3272220..bd464ecaf 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,7 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.cc + pcu.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -27,7 +27,7 @@ set(SOURCES # Package headers set(HEADERS PCU.h - pcu_io.h + pcu_io.h pcu_util.h reel/reel.h pcu_defines.h From 4d281e00ce0062ac61f39b58b754af85fd2ccd21 Mon Sep 17 00:00:00 2001 From: Jacob Merson Date: Fri, 31 Mar 2023 01:26:35 -0400 Subject: [PATCH 030/141] bump version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59d8fa390..019903f8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ endif() # This is the top level CMake file for the SCOREC build cmake_minimum_required(VERSION 3.8) -project(SCOREC VERSION 2.2.8 LANGUAGES CXX C) +project(SCOREC VERSION 2.3.0 LANGUAGES CXX C) include(cmake/bob.cmake) include(cmake/xsdk.cmake) From 412addcb1b0a1401c68f8e906ddf998d5a4a72b3 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 11 Jan 2024 13:44:34 -0500 Subject: [PATCH 031/141] added pcu_ as a variable of Mesh --- apf/apfMesh.cc | 1 + apf/apfMesh.h | 1 + pcu/pcu_util.h | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 6db2d67d5..e15362f20 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -166,6 +166,7 @@ void Mesh::init(FieldShape* s) baseP->init("coordinates",this,s,data); data->init(baseP); hasFrozenFields = false; + pcu_ = global_pcu } Mesh::~Mesh() diff --git a/apf/apfMesh.h b/apf/apfMesh.h index ac60009f5..16334e555 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -402,6 +402,7 @@ class Mesh std::vector fields; std::vector numberings; std::vector globalNumberings; + pcu::PCU* pcu_; }; /** \brief run consistency checks on an apf::Mesh structure diff --git a/pcu/pcu_util.h b/pcu/pcu_util.h index 8122599ad..d19f74b06 100644 --- a/pcu/pcu_util.h +++ b/pcu/pcu_util.h @@ -58,16 +58,16 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); } \ PCU_IGNORE_DIAGNOSTIC_END \ } while (0) -#define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ - do { \ - PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations)\ - if (! (cond)) { \ - char omsg[2048]; \ - sprintf(omsg, "%s failed at %s + %d \n %s", \ - #cond, __FILE__, __LINE__, msg); \ - PCU_Assert_Fail(omsg); \ - } \ - PCU_IGNORE_DIAGNOSTIC_END \ +#define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ + do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ + if (! (cond)) { \ + char omsg[2048]; \ + sprintf(omsg, "%s failed at %s + %d \n %s", \ + #cond, __FILE__, __LINE__, msg); \ + PCU_Assert_Fail(omsg); \ + } \ + PCU_IGNORE_DIAGNOSTIC_END \ } while(0) #ifdef NDEBUG From 63bf279ae7f41f61b7c2790ceb30f5980bdfe41b Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 11 Jan 2024 14:34:38 -0500 Subject: [PATCH 032/141] Added getPCU(), and beginning to change pcu functions --- apf/apfAdjReorder.cc | 4 ++-- apf/apfBoundaryToElementXi.cc | 2 +- apf/apfCavityOp.cc | 18 +++++++++--------- apf/apfMesh.h | 2 ++ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/apf/apfAdjReorder.cc b/apf/apfAdjReorder.cc index 73ed030cc..0973f5ca8 100644 --- a/apf/apfAdjReorder.cc +++ b/apf/apfAdjReorder.cc @@ -259,8 +259,8 @@ namespace apf { MeshTag* numbering = mesh->createIntTag(name,1); while (node_label >= 0) reorderConnected(mesh,numbering,node_label,element_label); - PCU_ALWAYS_ASSERT(node_label==-1); - PCU_ALWAYS_ASSERT(element_label==-1); + mesh->getPCU()->PCU_ALWAYS_ASSERT(node_label==-1); + mesh->getPCU()->PCU_ALWAYS_ASSERT(element_label==-1); return numbering; } diff --git a/apf/apfBoundaryToElementXi.cc b/apf/apfBoundaryToElementXi.cc index eada45ac0..0e0210ef1 100644 --- a/apf/apfBoundaryToElementXi.cc +++ b/apf/apfBoundaryToElementXi.cc @@ -57,7 +57,7 @@ Vector3 boundaryToElementXi( Vector3 exi(0,0,0); for (int i = 0; i < nbv; ++i) { int evi = findIn(ev, nev, bv[i]); - PCU_ALWAYS_ASSERT(evi >= 0); + m->getPCU()->PCU_ALWAYS_ASSERT(evi >= 0); exi += elem_vert_xi[et][evi] * shape_vals[i]; } return exi; diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 39b91b8d1..144c768df 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -134,10 +134,10 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = PCU_Min_Int(requests.empty()); + int done = mesh->getPCU()->PCU_Min_Int(requests.empty()); if (done) return false; /* throw in the local pull requests */ - int self = PCU_Comm_Self(); + int self = mesh->getPCU()->PCU_Comm_Self(); received.reserve(requests.size()); APF_ITERATE(Requests,requests,it) { @@ -147,7 +147,7 @@ bool CavityOp::sendPullRequests(std::vector& received) received.push_back(request); } /* now communicate the rest */ - PCU_Comm_Begin(); + mesh->getPCU()->PCU_Comm_Begin(); APF_ITERATE(Requests,requests,it) { CopyArray remotes; @@ -156,18 +156,18 @@ bool CavityOp::sendPullRequests(std::vector& received) { int remotePart = rit->peer; MeshEntity* remoteEntity = rit->entity; - PCU_COMM_PACK(remotePart,remoteEntity); + mesh->getPCU()->PCU_COMM_PACK(remotePart,remoteEntity); } } requests.clear(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + mesh->getPCU()->PCU_Comm_Send(); + while (mesh->getPCU()->PCU_Comm_Listen()) { PullRequest request; - request.to = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + request.to = mesh->getPCU()->PCU_Comm_Sender(); + while ( ! mesh->getPCU()->PCU_Comm_Unpacked()) { - PCU_COMM_UNPACK(request.e); + mesh->getPCU()->PCU_COMM_UNPACK(request.e); received.push_back(request); } } diff --git a/apf/apfMesh.h b/apf/apfMesh.h index 16334e555..5a627699f 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -395,6 +395,8 @@ class Mesh GlobalNumbering* findGlobalNumbering(const char* name); GlobalNumbering* getGlobalNumbering(int i); + /** \brief get the global pcu */ + PCU* getPCU() const {return pcu_;} /** \brief true if any associated fields use array storage */ bool hasFrozenFields; protected: From b3641184e3eb02eab9e2bf3a4050186fc24e45f7 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 16 Jan 2024 00:15:44 -0500 Subject: [PATCH 033/141] Can access global_pcu and some editing of function calls in /apf --- apf/apfAdjReorder.cc | 4 ++-- apf/apfBoundaryToElementXi.cc | 2 +- apf/apfCavityOp.cc | 19 +++++++++---------- apf/apfMesh.cc | 4 +++- apf/apfMesh.h | 4 +++- pcu/PCU.h | 3 +++ pcu/pcu.cc | 5 +++++ 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/apf/apfAdjReorder.cc b/apf/apfAdjReorder.cc index 0973f5ca8..73ed030cc 100644 --- a/apf/apfAdjReorder.cc +++ b/apf/apfAdjReorder.cc @@ -259,8 +259,8 @@ namespace apf { MeshTag* numbering = mesh->createIntTag(name,1); while (node_label >= 0) reorderConnected(mesh,numbering,node_label,element_label); - mesh->getPCU()->PCU_ALWAYS_ASSERT(node_label==-1); - mesh->getPCU()->PCU_ALWAYS_ASSERT(element_label==-1); + PCU_ALWAYS_ASSERT(node_label==-1); + PCU_ALWAYS_ASSERT(element_label==-1); return numbering; } diff --git a/apf/apfBoundaryToElementXi.cc b/apf/apfBoundaryToElementXi.cc index 0e0210ef1..eada45ac0 100644 --- a/apf/apfBoundaryToElementXi.cc +++ b/apf/apfBoundaryToElementXi.cc @@ -57,7 +57,7 @@ Vector3 boundaryToElementXi( Vector3 exi(0,0,0); for (int i = 0; i < nbv; ++i) { int evi = findIn(ev, nev, bv[i]); - m->getPCU()->PCU_ALWAYS_ASSERT(evi >= 0); + PCU_ALWAYS_ASSERT(evi >= 0); exi += elem_vert_xi[et][evi] * shape_vals[i]; } return exi; diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 144c768df..e89aa9c66 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -134,10 +134,10 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = mesh->getPCU()->PCU_Min_Int(requests.empty()); + int done = PCU_Min_Int(requests.empty()); if (done) return false; /* throw in the local pull requests */ - int self = mesh->getPCU()->PCU_Comm_Self(); + int self = mesh->getPCU()->Self(); received.reserve(requests.size()); APF_ITERATE(Requests,requests,it) { @@ -147,7 +147,7 @@ bool CavityOp::sendPullRequests(std::vector& received) received.push_back(request); } /* now communicate the rest */ - mesh->getPCU()->PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(Requests,requests,it) { CopyArray remotes; @@ -156,18 +156,17 @@ bool CavityOp::sendPullRequests(std::vector& received) { int remotePart = rit->peer; MeshEntity* remoteEntity = rit->entity; - mesh->getPCU()->PCU_COMM_PACK(remotePart,remoteEntity); - } + PCU_COMM_PACK(remotePart,remoteEntity); } } requests.clear(); - mesh->getPCU()->PCU_Comm_Send(); - while (mesh->getPCU()->PCU_Comm_Listen()) + mesh->getPCU()->Send(); + while (mesh->getPCU()->Listen()) { PullRequest request; - request.to = mesh->getPCU()->PCU_Comm_Sender(); - while ( ! mesh->getPCU()->PCU_Comm_Unpacked()) + request.to = mesh->getPCU()->Sender(); + while ( ! mesh->getPCU()->Unpacked()) { - mesh->getPCU()->PCU_COMM_UNPACK(request.e); + PCU_COMM_UNPACK(request.e); received.push_back(request); } } diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index e15362f20..22a561179 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -6,6 +6,8 @@ */ #include +#include +#include #include "apfCoordData.h" #include "apfVectorField.h" #include "apfShape.h" @@ -166,7 +168,7 @@ void Mesh::init(FieldShape* s) baseP->init("coordinates",this,s,data); data->init(baseP); hasFrozenFields = false; - pcu_ = global_pcu + pcu_ = PCU_GetGlobal(); } Mesh::~Mesh() diff --git a/apf/apfMesh.h b/apf/apfMesh.h index 5a627699f..e5680c133 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include "apfVector.h" #include "apfDynamicArray.h" @@ -396,7 +398,7 @@ class Mesh GlobalNumbering* getGlobalNumbering(int i); /** \brief get the global pcu */ - PCU* getPCU() const {return pcu_;} + pcu::PCU* getPCU() const {return pcu_;} /** \brief true if any associated fields use array storage */ bool hasFrozenFields; protected: diff --git a/pcu/PCU.h b/pcu/PCU.h index 02ff1255a..8b3e2c866 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -123,6 +123,9 @@ double PCU_Time(void); /*Memory usage*/ double PCU_GetMem(void); +/*Access global variable*/ + + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/pcu/pcu.cc b/pcu/pcu.cc index 0368ff6ff..95b24ffa0 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -563,4 +563,9 @@ double PCU_Time(void) { return pcu::Time(); } void PCU_Protect(void) { return pcu::Protect(); } double PCU_GetMem(void) { return pcu::GetMem(); } + + +/** \brief Return the global PCU + */ +pcu::PCU* PCU_GetGlobal(void) { return global_pcu; } } From 1010eb1beaee1f7adef26f95a899acb3484be8ff Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 16 Jan 2024 13:32:16 -0500 Subject: [PATCH 034/141] Changed functions in more files up to folder ma --- apf/apfCGNS.cc | 42 +++---- apf/apfConstruct.cc | 104 +++++++++--------- apf/apfConvertTags.h | 24 ++-- apf/apfFieldData.cc | 14 +-- apf/apfMesh.cc | 18 +-- apf/apfMigrate.cc | 74 ++++++------- apf/apfNumbering.cc | 6 +- apf/apfVerify.cc | 72 ++++++------ apf/apfVtk.cc | 8 +- apf/apfVtkPieceWiseFields.cc | 10 +- capstone_clis/capStoneAttachSolution.cc | 12 +- capstone_clis/capStoneAttachSolution2.cc | 12 +- .../capStoneAttachSolutionStrandOnly.cc | 12 +- capstone_clis/capStoneAttachSolutionUni.cc | 12 +- crv/crvVtk.cc | 28 ++--- dsp/dspGraphDistance.cc | 8 +- ma/maAdapt.cc | 18 +-- pcu/pcu.cc | 1 + 18 files changed, 238 insertions(+), 237 deletions(-) diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 4debbb58b..6225b1c0d 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -47,7 +47,7 @@ static Count count(apf::Mesh *m, int dim) { const int local = apf::countOwned(m, dim); int total = local; - PCU_Add_Ints(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array return std::make_pair(total, local); } @@ -290,7 +290,7 @@ void WriteFields(const CGNS &cgns, const std::vectorgetPCU()->Add(&size, 1); // size of total array // oddness of the api rmin[1] = rmin[0]; @@ -505,7 +505,7 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe m->end(cellIter); numbersByElementType[o] = counter; int total = counter; - PCU_Add_Ints(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array globalNumbersByElementType[o] = total; } cgsize_t allTotal = std::accumulate(globalNumbersByElementType.begin(), globalNumbersByElementType.end(), 0); @@ -546,12 +546,12 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe if (cgp_section_write(cgns.index, cgns.base, cgns.zone, name.c_str(), cgnsElementOrder[o], globalStart, globalEnd, 0, §ionNumber)) // global start, end within the file for that element type cgp_error_exit(); - std::vector allNumbersForThisType(PCU_Comm_Peers(), 0); + std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1, MPI_INT, PCU_Get_Comm()); cgsize_t num = 0; - for (int i = 0; i < PCU_Comm_Self(); i++) + for (int i = 0; i < m->getPCU()->Self(); i++) num += allNumbersForThisType[i]; cgsize_t elStart = globalStart + num; @@ -635,7 +635,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, int startOfBCBlock = startingLocation + 1; const int number = bc.second.size(); int total = number; - PCU_Add_Ints(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array if (total > 0) { const auto allEnd = startOfBCBlock + total - 1; //one-based @@ -657,12 +657,12 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, } } - std::vector allNumbersForThisType(PCU_Comm_Peers(), 0); + std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); cgsize_t num = 0; - for (int i = 0; i < PCU_Comm_Self(); i++) + for (int i = 0; i < m->getPCU()->Self(); i++) num += allNumbersForThisType[i]; cgsize_t elStart = startOfBCBlock + num; @@ -685,26 +685,26 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, cacheEnd = allEnd; } } - std::vector cacheStarts(PCU_Comm_Peers(), 0); + std::vector cacheStarts(m->getPCU()->Peers(), 0); MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1, - MPI_INT, PCU_Get_Comm()); - std::vector cacheEnds(PCU_Comm_Peers(), 0); + MPI_INT, m->getPCU()->GetMPIComm()); + std::vector cacheEnds(m->getPCU()->Peers(), 0); MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); return std::make_pair(cacheStarts, cacheEnds); }; const auto globalElementList = [](const std::vector &bcList, std::vector &allElements) { - std::vector sizes(PCU_Comm_Peers(), 0); // important initialiser + std::vector sizes(m->getPCU()->Peers(), 0); // important initialiser const int l = bcList.size(); MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); int totalLength = 0; for (const auto &i : sizes) totalLength += i; - std::vector displacement(PCU_Comm_Peers(), -1); + std::vector displacement(m->getPCU()->Peers(), -1); displacement[0] = 0; for (std::size_t i = 1; i < displacement.size(); i++) displacement[i] = displacement[i - 1] + sizes[i - 1]; @@ -712,7 +712,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, allElements.resize(totalLength); MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(), sizes.data(), displacement.data(), MPI_INT, - PCU_Get_Comm()); + m->getPCU()->GetMPIComm()); }; const auto doVertexBC = [&](const auto &iter) { @@ -755,7 +755,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, for (const auto &p : iter->second) { const auto se = BCEntityAdder(EdgeLoop, p, startingLocation); - for (int i = 0; i < PCU_Comm_Peers(); i++) + for (int i = 0; i < m->getPCU()->Peers(); i++) { PCU_ALWAYS_ASSERT_VERBOSE(se.first[i] == se.first[0], "Must all be the same "); PCU_ALWAYS_ASSERT_VERBOSE(se.second[i] == se.second[0], "Must all be the same "); @@ -779,7 +779,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, { const auto se = BCEntityAdder(FaceLoop, p, startingLocation); - for (int i = 0; i < PCU_Comm_Peers(); i++) + for (int i = 0; i < m->getPCU()->Peers(); i++) { PCU_ALWAYS_ASSERT_VERBOSE(se.first[i] == se.first[0], "Must all be the same "); PCU_ALWAYS_ASSERT_VERBOSE(se.second[i] == se.second[0], "Must all be the same "); @@ -1011,7 +1011,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap { static_assert(std::is_same::value, "cgsize_t not compiled as int"); - const auto myRank = PCU_Comm_Self(); + const auto myRank = m->getPCU()->Self(); const Count vertexCount = count(m, 0); const Count edgeCount = count(m, 1); const Count faceCount = count(m, 2); @@ -1048,7 +1048,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap sizes[2] = 0; // nodes are unsorted, as defined by api // Copy communicator - auto communicator = PCU_Get_Comm(); + auto communicator = m->getPCU()->GetMPIComm(); cgp_mpi_comm(communicator); // cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index 40fdbbeb0..e9bd33e7d 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -15,7 +15,7 @@ static void constructVerts( { ModelEntity* interior = m->findModelEntity(m->getDimension(), 0); int end = nelem * apf::Mesh::adjacentCount[etype][0]; - int self2 = PCU_Comm_Self(); + int self2 = m->getPCU()->Self(); for (int i = 0; i < end; ++i) if ( ! result.count(conn[i])) { result[conn[i]] = m->createVert_(interior); @@ -59,7 +59,7 @@ static Gid getMax(const GlobalToVert& globalToVert) static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) { Gid ifirst=0; - int self2 = PCU_Comm_Self(); + int self2 = m->getPCU()->Self(); APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; if(ifirst==0 || ifirst==13437400 ) { @@ -80,13 +80,13 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) ifirst++; } Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); Gid quotientL = total / peers; int quotient = quotientL; Gid remainderL = total % peers; int remainder = remainderL; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; if (self == (peers - 1)) lion_eprint(1, "CR1 mysize=%d \n",mySize); @@ -94,7 +94,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) TmpParts tmpParts(mySize); /* if we have a vertex, send its global id to the broker for that global id */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; if(gid < 0 || gid > max ){ @@ -104,22 +104,22 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) int tmpI=tmpL; int to = std::min(peers - 1,tmpI); PCU_COMM_PACK(to, gid); } - PCU_Comm_Send(); + m->getPCU()->Send(); Gid myOffset = (long)self * quotient; if (self == (peers - 1)) lion_eprint(1, "CR5: self=%d,myOffset=%ld,quotient=%d \n",self,myOffset,quotient); /* brokers store all the part ids that sent messages for each global id */ - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); Gid tmpL=gid - myOffset; // forcing 64 bit difference until we know it is safe int tmpI=tmpL; tmpParts.at(tmpI).push_back(from); } /* for each global id, send all associated part ids to all associated parts */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { @@ -132,11 +132,11 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) PCU_COMM_PACK(to, parts[k]); } } - PCU_Comm_Send(); + m->getPCU()->Send(); /* receiving a global id and associated parts, lookup the vertex and classify it on the partition model entity for that set of parts */ - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); int nparts; @@ -157,8 +157,8 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) pairs with parts in the residence of the vertex */ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert) { - int self = PCU_Comm_Self(); - PCU_Comm_Begin(); + int self = m->getPCU()->Self(); + m->getPCU()->Begin(); APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; if(gid < 0 ) @@ -172,18 +172,18 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert) PCU_COMM_PACK(*rit, vert); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); MeshEntity* remote; PCU_COMM_UNPACK(remote); - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); MeshEntity* vert = globalToVert[gid]; m->addRemote(vert, from, remote); } // who is not stuck? - lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self()); + lion_eprint(1, "%d done inside remotes \n",m->getPCU()->Self()); } NewElements assemble(Mesh2* m, const Gid* conn, int nelem, int etype, @@ -196,7 +196,7 @@ NewElements assemble(Mesh2* m, const Gid* conn, int nelem, int etype, void finalise(Mesh2* m, GlobalToVert& globalToVert) { constructResidence(m, globalToVert); - lion_eprint(1, "%d after residence \n",PCU_Comm_Self()); + lion_eprint(1, "%d after residence \n",m->getPCU()->Self()); constructRemotes(m, globalToVert); stitchMesh(m); m->acceptChanges(); @@ -215,11 +215,11 @@ void setCoords(Mesh2* m, const double* coords, int nverts, { Gid max = getMax(globalToVert); Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; Gid myOffset = (long)self * quotient; @@ -230,7 +230,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, Gid start = PCU_Exscan_Long(nverts); - PCU_Comm_Begin(); // the forced 64 bit math below may not be necessary + m->getPCU()->Begin(); // the forced 64 bit math below may not be necessary Gid tmpL=start / quotient; int tmpInt=tmpL; int to = std::min(peers - 1, tmpInt); @@ -250,7 +250,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, n = std::min(quotient, nverts); } PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(start); PCU_COMM_UNPACK(n); // |||||| more in-place 64 bit math PCU_Comm_Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double)); @@ -259,7 +259,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, /* Tell all the owners of the coords what we need */ typedef std::vector< std::vector > TmpParts; TmpParts tmpParts(mySize); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; Gid tmpL=gid / quotient; @@ -267,18 +267,18 @@ void setCoords(Mesh2* m, const double* coords, int nverts, int to = std::min(peers - 1, tmpInt); PCU_COMM_PACK(to, gid); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); - Gid from = PCU_Comm_Sender(); + Gid from = m->getPCU()->Sender(); Gid tmpL=gid - myOffset; int tmpInt=tmpL; tmpParts.at(tmpInt).push_back(from); } /* Send the coords to everybody who want them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { @@ -288,8 +288,8 @@ void setCoords(Mesh2* m, const double* coords, int nverts, PCU_Comm_Pack(to, &c[i*3], 3*sizeof(double)); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); double v[3]; @@ -306,11 +306,11 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, { Gid max = getMax(globalToVert); Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; Gid myOffset = (long)self * quotient; @@ -320,7 +320,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, Gid* c = new Gid[mySize]; Gid start = PCU_Exscan_Long(nverts); - PCU_Comm_Begin(); + m->getPCU()->Begin(); Gid tmpL=start / quotient; int tmpInt=tmpL; @@ -339,8 +339,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, to = std::min(peers - 1, to + 1); n = std::min(quotient, nverts); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(start); PCU_COMM_UNPACK(n); /// in-place 64 PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(Gid)); @@ -350,24 +350,24 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, /* Tell all the owners of the matches what we need */ typedef std::vector< std::vector > TmpParts; TmpParts tmpParts(mySize); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; int tmpI=gid / quotient; int to = std::min(peers - 1,tmpI); PCU_COMM_PACK(to, gid); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); tmpParts.at(gid - myOffset).push_back(from); } MeshTag* matchGidTag = m->createLongTag("matchGids", 1); /* Send the matches to everybody who wants them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { @@ -378,8 +378,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, PCU_COMM_PACK(to, matchGid); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); Gid match; @@ -394,7 +394,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, * the entity pointers and owners for mesh vertex gid [0..quotient), * process 1 holds gids [quotient..2*quotient), ... */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { MeshEntity* e = it->second; // KEJ does not follow this Gid gid = it->first; @@ -403,16 +403,16 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, PCU_COMM_PACK(to, gid); PCU_COMM_PACK(to, e); } - PCU_Comm_Send(); + m->getPCU()->Send(); typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs; typedef std::map< Gid, std::vector< EntOwnerPtrs > > GidPtrs; GidPtrs gidPtrs; - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); MeshEntity* vert; PCU_COMM_UNPACK(vert); - int owner = PCU_Comm_Sender(); + int owner = m->getPCU()->Sender(); gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert)); } @@ -420,7 +420,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, typedef std::pair MatchingPair; typedef std::map< MatchingPair, std::vector > MatchMap; MatchMap matchParts; - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { //loop over local verts Gid gid = it->first; Gid matchGid; @@ -432,20 +432,20 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, PCU_COMM_PACK(to, matchGid); // and the match gid needed } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); // request from entity gid Gid matchGid; PCU_COMM_UNPACK(matchGid); // requesting matched entity gid MatchingPair mp(gid,matchGid); - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); matchParts[mp].push_back(from); // store a list of the proceses that need the pair (entity gid, match gid) } /* Send the match pointer and owner process to everybody * who wants them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(MatchMap,matchParts,it) { MatchingPair mp = it->first; Gid gid = mp.first; @@ -466,8 +466,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, } } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; PCU_COMM_UNPACK(gid); Gid matchGid; diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h index 069caa43b..bdf916156 100644 --- a/apf/apfConvertTags.h +++ b/apf/apfConvertTags.h @@ -74,11 +74,11 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, { apf::Gid max = getMax(globalToVert); apf::Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; apf::Gid myOffset = (long)self * quotient; @@ -89,7 +89,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, apf::Gid start = PCU_Exscan_Long(nverts); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::Gid tmpL=start / quotient; int tmpI=tmpL; int to = std::min(peers - 1, tmpI); @@ -107,8 +107,8 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, to = std::min(peers - 1, to + 1); n = std::min(quotient, nverts); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(start); PCU_COMM_UNPACK(n); PCU_Comm_Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(T)); @@ -117,7 +117,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, /* Tell all the owners of the data what we need */ typedef std::vector< std::vector > TmpParts; TmpParts tmpParts(mySize); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { apf::Gid gid = it->first; tmpL=gid / quotient; @@ -126,16 +126,16 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, // replaced int to = std::min(peers - 1, gid / quotient); PCU_COMM_PACK(to, gid); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { apf::Gid gid; PCU_COMM_UNPACK(gid); - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); tmpParts.at(gid - myOffset).push_back(from); } /* Send the data to everybody who want them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { @@ -145,10 +145,10 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(T)); } } - PCU_Comm_Send(); + m->getPCU()->Send(); apf::MeshTag* t = createTag(m,tagName,entries); T* v = new T[entries]; - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { apf::Gid gid; PCU_COMM_UNPACK(gid); PCU_Comm_Unpack(v, entries*sizeof(T)); diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc index 2c3f1156a..2d1541b71 100644 --- a/apf/apfFieldData.cc +++ b/apf/apfFieldData.cc @@ -33,7 +33,7 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) continue; MeshEntity* e; MeshIterator* it = m->begin(d); - PCU_Comm_Begin(); + m->getPCU()->Begin(); while ((e = m->iterate(it))) { if (( ! data->hasEntity(e))|| @@ -58,8 +58,8 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; PCU_COMM_UNPACK(e); @@ -94,7 +94,7 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c MeshEntity* e; MeshIterator* it = m->begin(d); - PCU_Comm_Begin(); + m->getPCU()->Begin(); while ((e = m->iterate(it))) { /* send to all parts that can see this entity */ @@ -140,9 +140,9 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) - while ( ! PCU_Comm_Unpacked()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) + while ( ! m->getPCU()->Unpacked()) { /* receive and add. we only care about correctness on the owners */ MeshEntity* e; diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 22a561179..41ea506e2 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -972,7 +972,7 @@ bool MatchedSharing::isLess(Copy const& a, Copy const& b) Copy MatchedSharing::getOwnerCopy(MeshEntity* e) { - Copy owner(PCU_Comm_Self(), e); + Copy owner(mesh->getPCU()->Self(), e); CopyArray copies; this->getCopies(e, copies); APF_ITERATE(CopyArray, copies, cit) @@ -990,7 +990,7 @@ int MatchedSharing::getOwner(MeshEntity* e) bool MatchedSharing::isOwned(MeshEntity* e) { Copy owner = this->getOwnerCopy(e); - return owner.peer == PCU_Comm_Self() && owner.entity == e; + return owner.peer == mesh->getPCU()->Self() && owner.entity == e; } void MatchedSharing::getCopies(MeshEntity* e, @@ -1012,23 +1012,23 @@ void MatchedSharing::getNeighbors(Parts& neighbors) neighbors.insert(cit->peer); } mesh->end(it); - neighbors.erase(PCU_Comm_Self()); + neighbors.erase(mesh->getPCU()->Self()); } void MatchedSharing::formCountMap() { size_t count = mesh->count(mesh->getDimension()); - countMap[PCU_Comm_Self()] = count; - PCU_Comm_Begin(); + countMap[mesh->getPCU()->Self()] = count; + mesh->getPCU()->Begin(); Parts neighbors; getNeighbors(neighbors); APF_ITERATE(Parts, neighbors, nit) PCU_COMM_PACK(*nit, count); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + mesh->getPCU()->Send(); + while (mesh->getPCU()->Receive()) { size_t oc; PCU_COMM_UNPACK(oc); - countMap[PCU_Comm_Sender()] = oc; + countMap[mesh->getPCU()->Sender()] = oc; } } @@ -1036,7 +1036,7 @@ bool MatchedSharing::isShared(MeshEntity* e) { CopyArray copies; this->getCopies(e, copies); APF_ITERATE(CopyArray, copies, it) - if (it->peer != PCU_Comm_Self()) + if (it->peer != mesh->getPCU()->Self()) return true; return false; } diff --git a/apf/apfMigrate.cc b/apf/apfMigrate.cc index a0df01481..caeb1e8f6 100644 --- a/apf/apfMigrate.cc +++ b/apf/apfMigrate.cc @@ -28,7 +28,7 @@ static void getAffected( EntityVector affected[4]) { int maxDimension = m->getDimension(); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); affected[maxDimension].reserve(plan->count()); for (int i=0; i < plan->count(); ++i) { @@ -43,7 +43,7 @@ static void getAffected( for (int dimension=maxDimension-1; dimension >= 0; --dimension) { int upDimension = dimension + 1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[upDimension],it) { MeshEntity* up = *it; @@ -69,8 +69,8 @@ static void getAffected( } }//downward adjacent loop }//upward affected loop - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* entity; PCU_COMM_UNPACK(entity); @@ -117,7 +117,7 @@ void reduceMatchingToSenders( EntityVector senders[4]) { if ( ! m->hasMatching()) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d=0; d < 4; ++d) { for (size_t i=0; i < senders[d].size(); ++i) @@ -135,11 +135,11 @@ void reduceMatchingToSenders( m->clearMatches(e); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { - int sender = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int sender = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { MeshEntity* e; PCU_COMM_UNPACK(e); @@ -203,7 +203,7 @@ static void updateResidences( } for (int dimension = maxDimension-1; dimension >= 0; --dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[dimension],it) { MeshEntity* entity = *it; @@ -226,8 +226,8 @@ static void updateResidences( packParts(rit->first,newResidence); } } - PCU_Comm_Send(); - while(PCU_Comm_Receive()) + m->getPCU()->Send(); + while(m->getPCU()->Receive()) { MeshEntity* entity; PCU_COMM_UNPACK(entity); @@ -493,7 +493,7 @@ static MeshEntity* unpackEntity( Mesh2* m, DynamicArray& tags) { - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); int type; PCU_COMM_UNPACK(type); MeshEntity* sender; @@ -539,7 +539,7 @@ static void receiveEntities( EntityVector& received) { received.reserve(1024); - while (PCU_Comm_Receive()) + while (m->getPCU()->Receive()) received.push_back(unpackEntity(m,tags)); } @@ -564,10 +564,10 @@ static void echoRemotes( static void receiveRemotes(Mesh2* m) { - while (PCU_Comm_Listen()) + while (m->getPCU()->Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { MeshEntity* sender; PCU_COMM_UNPACK(sender); @@ -596,7 +596,7 @@ static void getNewCopies( APF_ITERATE(Copies,allRemotes,it) if (residence.count(it->first)) newCopies.insert(*it); - int rank = PCU_Comm_Self(); + int rank = m->getPCU()->Self(); if (residence.count(rank)) newCopies[rank]=e; } @@ -634,8 +634,8 @@ static void bcastRemotes( Mesh2* m, EntityVector& senders) { - PCU_Comm_Begin(); - int rank = PCU_Comm_Self(); + m->getPCU()->Begin(); + int rank = m->getPCU()->Self(); APF_ITERATE(EntityVector,senders,it) { MeshEntity* e = *it; @@ -651,8 +651,8 @@ static void bcastRemotes( newCopies.erase(rank); m->setRemotes(e,newCopies); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; PCU_COMM_UNPACK(e); @@ -668,9 +668,9 @@ static void setupRemotes( EntityVector& received, EntityVector& senders) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); echoRemotes(m,received); - PCU_Comm_Send(); + m->getPCU()->Send(); receiveRemotes(m); bcastRemotes(m,senders); } @@ -684,9 +684,9 @@ void moveEntities( int maxDimension = m->getDimension(); for (int dimension = 0; dimension <= maxDimension; ++dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); sendEntities(m,senders[dimension],tags); - PCU_Comm_Send(); + m->getPCU()->Send(); EntityVector received; receiveEntities(m,tags,received); setupRemotes(m,received,senders[dimension]); @@ -704,8 +704,8 @@ static void updateSenderMatching( EntityVector affected[4], EntityVector senders[4]) { - PCU_Comm_Begin(); - int self = PCU_Comm_Self(); + m->getPCU()->Begin(); + int self = m->getPCU()->Self(); for (int d=0; d < 4; ++d) { for (size_t i=0; i < senders[d].size(); ++i) @@ -736,8 +736,8 @@ static void updateSenderMatching( for (size_t i=0; i < affected[d].size(); ++i) m->clearMatches(affected[d][i]); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; PCU_COMM_UNPACK(e); @@ -753,8 +753,8 @@ static void bcastMatching( Mesh2* m, EntityVector senders[4]) { - PCU_Comm_Begin(); - int self = PCU_Comm_Self(); + m->getPCU()->Begin(); + int self = m->getPCU()->Self(); for (int d=0; d < 4; ++d) { for (size_t i=0; i < senders[d].size(); ++i) @@ -788,8 +788,8 @@ static void bcastMatching( } } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; PCU_COMM_UNPACK(e); @@ -826,7 +826,7 @@ void deleteOldEntities( Mesh2* m, EntityVector affected[4]) { - int rank = PCU_Comm_Self(); + int rank = m->getPCU()->Self(); int maxDimension = m->getDimension(); for (int d=maxDimension; d >= 0; --d) APF_ITERATE(EntityVector,affected[d],it) @@ -883,7 +883,7 @@ static void migrate2(Mesh2* m, Migration* plan) } delete plan; size_t sent = 0; - while (PCU_Or(sent < tmp.size())) + while (m->getPCU()->Or(sent < tmp.size())) { plan = new Migration(m); size_t send = std::min(tmp.size() - sent, migrationLimit); @@ -896,7 +896,7 @@ static void migrate2(Mesh2* m, Migration* plan) void migrateSilent(Mesh2* m, Migration* plan) { - if (PCU_Or(static_cast(plan->count()) > migrationLimit)) + if (m->getPCU()->Or(static_cast(plan->count()) > migrationLimit)) migrate2(m, plan); else migrate1(m, plan); diff --git a/apf/apfNumbering.cc b/apf/apfNumbering.cc index 5d62935e4..3130aed9e 100644 --- a/apf/apfNumbering.cc +++ b/apf/apfNumbering.cc @@ -401,7 +401,7 @@ static void synchronizeEntitySet( Mesh* m, EntitySet& set) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntitySet,set,it) if (m->isShared(*it)) { @@ -410,8 +410,8 @@ static void synchronizeEntitySet( APF_ITERATE(Copies,remotes,rit) PCU_COMM_PACK(rit->first,rit->second); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; PCU_COMM_UNPACK(e); diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc index 448b1ee41..5c2ad968b 100644 --- a/apf/apfVerify.cc +++ b/apf/apfVerify.cc @@ -212,7 +212,7 @@ static Copies getAllCopies(Mesh* m, MeshEntity* e) { Copies c; m->getRemotes(e, c); - c[PCU_Comm_Self()] = e; + c[m->getPCU()->Self()] = e; return c; } @@ -261,7 +261,7 @@ static void sendAllCopies(Mesh* m, MeshEntity* e) a = getAllCopies(m, e); verifyAllCopies(a); m->getRemotes(e, r); - PCU_ALWAYS_ASSERT(!r.count(PCU_Comm_Self())); + PCU_ALWAYS_ASSERT(!r.count(m->getPCU()->Self())); APF_ITERATE(Copies, r, it) { PCU_COMM_PACK(it->first, it->second); @@ -283,15 +283,15 @@ static void verifyRemoteCopies(Mesh* m) { for (int d = 0; d <= m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); MeshEntity* e; while ((e = m->iterate(it))) if (m->isShared(e) && !m->isGhost(e)) sendAllCopies(m, e); m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveAllCopies(m); } } @@ -311,7 +311,7 @@ static void sendGhostCopies(Mesh* m, MeshEntity* e) static void receiveGhostCopies(Mesh* m) { - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); MeshEntity* e; PCU_COMM_UNPACK(e); MeshEntity* g; @@ -331,7 +331,7 @@ static void receiveGhostCopies(Mesh* m) static void verifyGhostCopies(Mesh* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d = 0; d <= m->getDimension(); ++d) { MeshIterator* it = m->begin(d); @@ -341,8 +341,8 @@ static void verifyGhostCopies(Mesh* m) sendGhostCopies(m, e); m->end(it); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveGhostCopies(m); } @@ -377,7 +377,7 @@ static void receiveMatches(Mesh* m) PCU_COMM_UNPACK(e); Matches matches; m->getMatches(e, matches); - PCU_ALWAYS_ASSERT(hasMatch(matches, PCU_Comm_Sender(), source)); + PCU_ALWAYS_ASSERT(hasMatch(matches, m->getPCU()->Sender(), source)); } static bool hasDuplicates(Matches const& matches) { @@ -391,7 +391,7 @@ static bool hasDuplicates(Matches const& matches) { static void verifyMatches(Mesh* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d = 0; d <= m->getDimension(); ++d) { MeshIterator* it = m->begin(d); @@ -404,8 +404,8 @@ static void verifyMatches(Mesh* m) } m->end(it); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveMatches(m); } @@ -443,16 +443,16 @@ static bool receiveCoords(Mesh* m) static long verifyCoords(Mesh* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(0); MeshEntity* e; while ((e = m->iterate(it))) if (m->isShared(e)) sendCoords(m, e); m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); long n = 0; - while (PCU_Comm_Receive()) + while (m->getPCU()->Receive()) if (!receiveCoords(m)) ++n; return PCU_Add_Long(n); @@ -522,15 +522,15 @@ static void receiveAlignment(Mesh* m) static void verifyAlignment(Mesh* m) { for (int d = 1; d <= m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); MeshEntity* e; while ((e = m->iterate(it))) if (m->isShared(e)) sendAlignment(m, e); m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveAlignment(m); } } @@ -556,22 +556,22 @@ void unpackFieldInfo(std::string& name, int& type, int& size) static void verifyFields(Mesh* m) { - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); int n = m->countFields(); std::vector fields; for (int i=0; igetField(i)); if (!fields.size()) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); if (self) { PCU_COMM_PACK(self - 1, n); for (int i = 0; i < n; ++i) packFieldInfo(fields[i], self - 1); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int n; PCU_COMM_UNPACK(n); PCU_ALWAYS_ASSERT(fields.size() == (size_t)n); @@ -637,7 +637,7 @@ static void sendTagData(Mesh* m, MeshEntity* e, DynamicArray& tags, Co } default: break; } - PCU_Comm_Write(it->first, (void*)msg_send, msg_size); + m->getPCU()->Write(it->first, (void*)msg_send, msg_size); free(msg_send); } // apf_iterate } // for @@ -653,7 +653,7 @@ static void receiveTagData(Mesh* m, DynamicArray& tags) size_t msg_size; std::set mismatch_tags; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { e = *((MeshEntity**)msg_recv); int *id = (int*)((char*)msg_recv+sizeof(MeshEntity*)); @@ -722,7 +722,7 @@ static void receiveTagData(Mesh* m, DynamicArray& tags) } // while int global_size = PCU_Max_Int((int)mismatch_tags.size()); - if (global_size&&!PCU_Comm_Self()) + if (global_size&&!m->getPCU()->Self()) for (std::set::iterator it=mismatch_tags.begin(); it!=mismatch_tags.end(); ++it) lion_oprint(1," - tag \"%s\" data mismatch over remote/ghost copies\n", m->getTagName(*it)); } @@ -731,11 +731,11 @@ static void verifyTags(Mesh* m) { DynamicArray tags; m->getTags(tags); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); int n = tags.getSize(); if (!n) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); if (self) { PCU_COMM_PACK(self - 1, n); for (int i = 0; i < n; ++i) @@ -754,8 +754,8 @@ static void verifyTags(Mesh* m) lion_oprint(1,"\n"); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int n; PCU_COMM_UNPACK(n); PCU_ALWAYS_ASSERT(tags.getSize() == (size_t)n); @@ -774,12 +774,12 @@ static void verifyTags(Mesh* m) for (int d = 0; d <= m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); MeshEntity* e; while ((e = m->iterate(it))) { - if (m->getOwner(e)!=PCU_Comm_Self()) continue; + if (m->getOwner(e)!=m->getPCU()->Self()) continue; if (m->isShared(e)) { Copies r; m->getRemotes(e, r); @@ -792,7 +792,7 @@ static void verifyTags(Mesh* m) } } // while m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); receiveTagData(m, tags); } // for } @@ -828,13 +828,13 @@ void verify(Mesh* m, bool abort_on_error) verifyAlignment(m); verifyMatches(m); long n = verifyCoords(m); - if (n && (!PCU_Comm_Self())) + if (n && (!m->getPCU()->Self())) lion_eprint(1,"apf::verify fail: %ld coordinate mismatches\n", n); n = verifyVolumes(m); - if (n && (!PCU_Comm_Self())) + if (n && (!m->getPCU()->Self())) lion_eprint(1,"apf::verify warning: %ld negative simplex elements\n", n); double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"mesh verified in %f seconds\n", t1 - t0); } diff --git a/apf/apfVtk.cc b/apf/apfVtk.cc index e71744a38..98f09985b 100644 --- a/apf/apfVtk.cc +++ b/apf/apfVtk.cc @@ -929,18 +929,18 @@ void writeVtkFilesRunner(const char* prefix, { if (cellDim == -1) cellDim = m->getDimension(); double t0 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) { safe_mkdir(prefix); - makeVtuSubdirectories(prefix, PCU_Comm_Peers()); + makeVtuSubdirectories(prefix, m->getPCU()->Peers()); writePvtuFile(prefix, m, writeFields, isWritingBinary, cellDim); } - PCU_Barrier(); + m->getPCU()->Barrier(); Numbering* n = numberOverlapNodes(m,"apf_vtk_number"); m->removeNumbering(n); writeVtuFile(prefix, n, writeFields, isWritingBinary, cellDim); double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) { lion_oprint(1,"vtk files %s written in %f seconds\n", prefix, t1 - t0); } diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc index 35016f775..4879d4cc0 100644 --- a/apf/apfVtkPieceWiseFields.cc +++ b/apf/apfVtkPieceWiseFields.cc @@ -101,8 +101,8 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, apf::Vector3(0., 0., 1.) }; - std::string fileName = getPieceFileName(PCU_Comm_Self()); - std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, PCU_Comm_Self()); + std::string fileName = getPieceFileName(m->getPCU()->Self()); + std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, m->getPCU()->Self()); std::stringstream buf; buf << "# vtk DataFile Version 3.0\n" @@ -163,7 +163,7 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, buf << "LOOKUP_TABLE default\n"; it = m->begin(m->getDimension()); while( (e = m->iterate(it)) ) { - buf << PCU_Comm_Self() << '\n'; + buf << m->getPCU()->Self() << '\n'; } m->end(it); @@ -191,7 +191,7 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, m->end(it); } double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile into buffers: %f seconds\n", t1 - t0); } @@ -201,7 +201,7 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, file << buf.rdbuf(); } double t2 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) { lion_oprint(1,"writeNedelecVtkFile buffers to disk: %f seconds\n", t2 - t1); } diff --git a/capstone_clis/capStoneAttachSolution.cc b/capstone_clis/capStoneAttachSolution.cc index 28b850a78..0d8e7f647 100644 --- a/capstone_clis/capStoneAttachSolution.cc +++ b/capstone_clis/capStoneAttachSolution.cc @@ -249,11 +249,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -265,7 +265,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { PCU_COMM_UNPACK(ent); PCU_COMM_UNPACK(receivedSize); @@ -295,7 +295,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.push(ent); } - PCU_Comm_Begin(); + m->getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -310,9 +310,9 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack PCU_COMM_UNPACK(ent); diff --git a/capstone_clis/capStoneAttachSolution2.cc b/capstone_clis/capStoneAttachSolution2.cc index 7d73b0592..797daecdd 100644 --- a/capstone_clis/capStoneAttachSolution2.cc +++ b/capstone_clis/capStoneAttachSolution2.cc @@ -221,11 +221,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -237,7 +237,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { PCU_COMM_UNPACK(ent); PCU_COMM_UNPACK(receivedSize); @@ -267,7 +267,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.push(ent); } - PCU_Comm_Begin(); + m->getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -282,9 +282,9 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack PCU_COMM_UNPACK(ent); diff --git a/capstone_clis/capStoneAttachSolutionStrandOnly.cc b/capstone_clis/capStoneAttachSolutionStrandOnly.cc index e39994f52..8b2ffdae9 100644 --- a/capstone_clis/capStoneAttachSolutionStrandOnly.cc +++ b/capstone_clis/capStoneAttachSolutionStrandOnly.cc @@ -247,11 +247,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -263,7 +263,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { PCU_COMM_UNPACK(ent); PCU_COMM_UNPACK(receivedSize); @@ -293,7 +293,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.push(ent); } - PCU_Comm_Begin(); + m->getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -308,9 +308,9 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack PCU_COMM_UNPACK(ent); diff --git a/capstone_clis/capStoneAttachSolutionUni.cc b/capstone_clis/capStoneAttachSolutionUni.cc index 504cc9e03..416f1e41c 100644 --- a/capstone_clis/capStoneAttachSolutionUni.cc +++ b/capstone_clis/capStoneAttachSolutionUni.cc @@ -225,11 +225,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -241,7 +241,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { PCU_COMM_UNPACK(ent); PCU_COMM_UNPACK(receivedSize); @@ -271,7 +271,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.push(ent); } - PCU_Comm_Begin(); + m->getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -286,9 +286,9 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack PCU_COMM_UNPACK(ent); diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc index dd72bafac..1d2b352f0 100644 --- a/crv/crvVtk.cc +++ b/crv/crvVtk.cc @@ -737,7 +737,7 @@ static void writePvtuFile(const char* prefix, const char* suffix, } writePPointData(file,m); file << "\n"; - for (int i=0; i < PCU_Comm_Peers(); ++i) + for (int i=0; i < m->getPCU()->Peers(); ++i) { std::stringstream ssPart; ssPart << "vtu/" @@ -764,13 +764,13 @@ static void writePointData(std::ostream& file, apf::Mesh* m, void writeInterpolationPointVtuFiles(apf::Mesh* m, const char* prefix) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) writePvtuFile(prefix,"_interPts",m,apf::Mesh::VERTEX); - PCU_Barrier(); + m->getPCU()->Barrier(); std::stringstream ss; - ss << prefix << PCU_Comm_Self() << "_interPts" + ss << prefix << m->getPCU()->Self() << "_interPts" << "_" << m->getShape()->getOrder() << ".vtu"; @@ -831,18 +831,18 @@ void writeInterpolationPointVtuFiles(apf::Mesh* m, const char* prefix) file << buf.rdbuf(); } - PCU_Barrier(); + m->getPCU()->Barrier(); } void writeControlPointVtuFiles(apf::Mesh* m, const char* prefix) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) writePvtuFile(prefix,getSuffix(apf::Mesh::VERTEX),m,apf::Mesh::VERTEX); - PCU_Barrier(); + m->getPCU()->Barrier(); std::stringstream ss; - ss << prefix << PCU_Comm_Self() + ss << prefix << m->getPCU()->Self() << getSuffix(apf::Mesh::VERTEX) << "_" << m->getShape()->getOrder() << ".vtu"; @@ -901,7 +901,7 @@ void writeControlPointVtuFiles(apf::Mesh* m, const char* prefix) file << buf.rdbuf(); } - PCU_Barrier(); + m->getPCU()->Barrier(); } static void safe_mkdir(const char* path) @@ -1002,16 +1002,16 @@ void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix) void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix) { double t0 = PCU_Time(); - if (!PCU_Comm_Self()) { + if (!m->getPCU()->Self()) { makeDirectories(prefix, type, n); writePvtuFile(getPvtuDirectoryStr(prefix, type, n).c_str(),"",m,type); } - PCU_Barrier(); + m->getPCU()->Barrier(); std::stringstream ss; ss << getVtuDirectoryStr(prefix, type, n) << "/order_" << m->getShape()->getOrder() << "_" - << PCU_Comm_Self() + << m->getPCU()->Self() << ".vtu"; std::string fileName = ss.str(); std::stringstream buf; @@ -1055,9 +1055,9 @@ void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix) file << buf.rdbuf(); } - PCU_Barrier(); + m->getPCU()->Barrier(); double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"%s vtk files %s written in %f seconds\n", apf::Mesh::typeName[type],getPvtuDirectoryStr(prefix, type, n).c_str(),t1 - t0); } diff --git a/dsp/dspGraphDistance.cc b/dsp/dspGraphDistance.cc index 4b2ec8eb7..a73d3708f 100644 --- a/dsp/dspGraphDistance.cc +++ b/dsp/dspGraphDistance.cc @@ -39,7 +39,7 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, } } m->end(it); - for (int layer = 0; PCU_Or(!empty(vs, first)); ++layer) { + for (int layer = 0; m->getPCU()->Or(!empty(vs, first)); ++layer) { size_t layer_end = vs.size(); while (first < layer_end) { v = pop(vs, first); @@ -56,7 +56,7 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, } } } - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshEntity* sv; for (size_t i = first; i < vs.size(); ++i) { sv = vs[i]; @@ -69,8 +69,8 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, APF_ITERATE(apf::Copies, remotes, rit) PCU_COMM_PACK(rit->first, rit->second); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(sv); if (!apf::isNumbered(dst, sv, 0, 0)) { apf::number(dst, sv, 0, 0, layer + 1); diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index 606e095d5..406b090ae 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -111,7 +111,7 @@ void setFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == mesh->getPCU()->Self()); setFlag(a, it->entity, flag); } } @@ -131,7 +131,7 @@ void clearFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == mesh->getPCU()->Self()); clearFlag(a, it->entity, flag); } } @@ -228,7 +228,7 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) { Mesh* m = a->mesh; apf::Sharing* sh = apf::getSharing(m); - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); Entity* e; Iterator* it = m->begin(dimension); while ((e = m->iterate(it))) { @@ -243,9 +243,9 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) } } m->end(it); - PCU_Comm_Send(); + mesh->getPCU()->Send(); bool ok = true; - while (PCU_Comm_Receive()) { + while (mesh->getPCU()->Receive()) { PCU_COMM_UNPACK(e); bool value; PCU_COMM_UNPACK(value); @@ -473,7 +473,7 @@ void clearBuildCallback(Adapt* a) void print(const char* format, ...) { - if (PCU_Comm_Self()) + if (mesh->getPCU()->Self()) return; lion_oprint(1,"\nMeshAdapt: "); va_list ap; @@ -500,7 +500,7 @@ void syncFlag(Adapt* a, int dimension, int flag) { Mesh* m = a->mesh; apf::Sharing* sh = apf::getSharing(m); - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); Entity* e; Iterator* it = m->begin(dimension); while ((e = m->iterate(it))) { @@ -512,8 +512,8 @@ void syncFlag(Adapt* a, int dimension, int flag) PCU_COMM_PACK(rit->peer, rit->entity); } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + mesh->getPCU()->Send(); + while (mesh->getPCU()->Receive()) { PCU_COMM_UNPACK(e); setFlag(a,e,flag); } diff --git a/pcu/pcu.cc b/pcu/pcu.cc index 95b24ffa0..bddff5c70 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -568,4 +568,5 @@ double PCU_GetMem(void) { return pcu::GetMem(); } /** \brief Return the global PCU */ pcu::PCU* PCU_GetGlobal(void) { return global_pcu; } + } From 041b6f0fa99315edfeb96695eb3dae2936ec10c5 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 17 Jan 2024 13:41:30 -0500 Subject: [PATCH 035/141] More PCU changes --- config.sh | 6 ++++++ ma/maAdapt.cc | 18 +++++++++--------- ma/maCrawler.cc | 10 +++++----- ma/maExtrude.cc | 8 ++++---- ma/maLayerCoarsen.cc | 10 +++++----- ma/maLayerSnap.cc | 12 ++++++------ ma/maMatch.cc | 10 +++++----- ma/maMatchedCollapse.cc | 8 ++++---- ma/maMatchedSnapper.cc | 2 +- ma/maRefine.cc | 14 +++++++------- ma/maSnap.cc | 4 ++-- 11 files changed, 54 insertions(+), 48 deletions(-) create mode 100644 config.sh diff --git a/config.sh b/config.sh new file mode 100644 index 000000000..21394a02a --- /dev/null +++ b/config.sh @@ -0,0 +1,6 @@ +cmake -S . -B /lore/mccalf/build \ + -DCMAKE_C_COMPILER=mpicc \ + -DCMAKE_CXX_COMPILER=mpicxx \ + -DIS_TESTING=on \ + -DBUILD_EXES=on \ + -DMESHES=/users/mccalf/pcu-update/pumi-meshes diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index 406b090ae..f56b9ea4b 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -111,7 +111,7 @@ void setFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == mesh->getPCU()->Self()); + PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); setFlag(a, it->entity, flag); } } @@ -131,7 +131,7 @@ void clearFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == mesh->getPCU()->Self()); + PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); clearFlag(a, it->entity, flag); } } @@ -228,7 +228,7 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) { Mesh* m = a->mesh; apf::Sharing* sh = apf::getSharing(m); - mesh->getPCU()->Begin(); + m->getPCU()->Begin(); Entity* e; Iterator* it = m->begin(dimension); while ((e = m->iterate(it))) { @@ -243,9 +243,9 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) } } m->end(it); - mesh->getPCU()->Send(); + m->getPCU()->Send(); bool ok = true; - while (mesh->getPCU()->Receive()) { + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(e); bool value; PCU_COMM_UNPACK(value); @@ -473,7 +473,7 @@ void clearBuildCallback(Adapt* a) void print(const char* format, ...) { - if (mesh->getPCU()->Self()) + if (PCU_Comm_Self()) return; lion_oprint(1,"\nMeshAdapt: "); va_list ap; @@ -500,7 +500,7 @@ void syncFlag(Adapt* a, int dimension, int flag) { Mesh* m = a->mesh; apf::Sharing* sh = apf::getSharing(m); - mesh->getPCU()->Begin(); + m->getPCU()->Begin(); Entity* e; Iterator* it = m->begin(dimension); while ((e = m->iterate(it))) { @@ -512,8 +512,8 @@ void syncFlag(Adapt* a, int dimension, int flag) PCU_COMM_PACK(rit->peer, rit->entity); } m->end(it); - mesh->getPCU()->Send(); - while (mesh->getPCU()->Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(e); setFlag(a,e,flag); } diff --git a/ma/maCrawler.cc b/ma/maCrawler.cc index d4c0f6b01..2bac53d10 100644 --- a/ma/maCrawler.cc +++ b/ma/maCrawler.cc @@ -9,7 +9,7 @@ namespace ma { void syncLayer(Crawler* c, Crawler::Layer& layer) { Mesh* m = c->mesh; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (size_t i = 0; i < layer.size(); ++i) { Entity* e = layer[i]; if (m->isShared(e)) { @@ -21,10 +21,10 @@ void syncLayer(Crawler* c, Crawler::Layer& layer) } } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { Entity* e; PCU_COMM_UNPACK(e); if (c->recv(e, from)) diff --git a/ma/maExtrude.cc b/ma/maExtrude.cc index 2fb42c897..2ee63f3ff 100644 --- a/ma/maExtrude.cc +++ b/ma/maExtrude.cc @@ -293,7 +293,7 @@ class DebugBuildCallback : public apf::BuildCallback { void stitchVerts(Mesh* m, Crawler::Layer const& prev_verts, Crawler::Layer const& next_verts, Tag* indices) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); PCU_ALWAYS_ASSERT(prev_verts.size() == next_verts.size()); for (size_t i = 0; i < prev_verts.size(); ++i) { Entity* prev_vert = prev_verts[i]; @@ -307,9 +307,9 @@ void stitchVerts(Mesh* m, Crawler::Layer const& prev_verts, PCU_COMM_PACK(remote_part, next_vert); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - int remote_part = PCU_Comm_Sender(); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + int remote_part = m->getPCU()->Sender(); Entity* prev_vert; Entity* remote_next_vert; PCU_COMM_UNPACK(prev_vert); diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index 12feced3d..b6611eb33 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -161,8 +161,8 @@ static apf::Migration* planLayerCollapseMigration(Adapt* a, int d, int round) (m->getModelType(m->toModel(e)) == d)) { Entity* v[2]; m->getDownward(e, 0, v); - cl.handle(v[0], PCU_Comm_Self()); - cl.handle(v[1], PCU_Comm_Self()); + cl.handle(v[0], m->getPCU()->Self()); + cl.handle(v[1], m->getPCU()->Self()); } m->end(it); crawlLayers(&cl); @@ -172,13 +172,13 @@ static apf::Migration* planLayerCollapseMigration(Adapt* a, int d, int round) static bool wouldEmptyParts(apf::Migration* plan) { apf::Mesh* m = plan->getMesh(); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); size_t sendingAway = 0; for (int i = 0; i < plan->count(); ++i) if (plan->sending(plan->get(i)) != self) ++sendingAway; bool wouldEmptyThisPart = (sendingAway == m->count(m->getDimension())); - return PCU_Or(wouldEmptyThisPart); + return m->getPCU()->Or(wouldEmptyThisPart); } static void migrateForLayerCollapse(Adapt* a, int d, int round) @@ -258,7 +258,7 @@ static long collapseAllStacks(Adapt* a, int d) collapseLocalStacks(a, skipCount, successCount, failureCount, d); allSuccesses += successCount; ++round; - } while (PCU_Or(skipCount)); + } while (a->mesh->getPCU()->Or(skipCount)); return PCU_Add_Long(allSuccesses); } diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index 1fc249800..0c64e6448 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -134,7 +134,7 @@ struct BaseTopLinker : public Crawler void begin(Layer& first) { getDimensionBase(a, 0, first); - int peer = PCU_Comm_Self(); + int peer = m->getPCU()->Self(); for (size_t i = 0; i < first.size(); ++i) { if (!m->isOwned(first[i])) continue; @@ -387,7 +387,7 @@ static void crawlLayers_doubleSync(Crawler* c) { Crawler::Layer layer; c->begin(layer); - while (PCU_Or( ! layer.empty())) { + while (c->mesh->getPCU()->Or( ! layer.empty())) { crawlLayer(c, layer); syncLayer(c, layer); syncLayer(c, layer); @@ -400,7 +400,7 @@ static bool checkForUnsnap(Adapt* a, Tag* snapTag) double t0 = PCU_Time(); UnsnapChecker op(a, snapTag); crawlLayers_doubleSync(&op); - bool notOk = PCU_Or(op.foundAnything); + bool notOk = a->mesh->getPCU()->Or(op.foundAnything); double t1 = PCU_Time(); if (notOk) print("checked snapped curves in %f seconds, found some to unsnap", t1 - t0); @@ -417,7 +417,7 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) Mesh* m = l.m; long n = 0; Entity* v; - PCU_Comm_Begin(); + m->getPCU()->Begin(); Iterator* it = m->begin(0); while ((v = m->iterate(it))) if (getFlag(a, v, LAYER_TOP) && @@ -429,8 +429,8 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) ++n; } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int link; PCU_COMM_UNPACK(link); Entity* v = l.lookup(link); diff --git a/ma/maMatch.cc b/ma/maMatch.cc index cd28b47bc..0043ca05d 100644 --- a/ma/maMatch.cc +++ b/ma/maMatch.cc @@ -36,7 +36,7 @@ void matchNewElements(Refine* r) long face_count = 0; for (int d=1; d < m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (size_t i=0; i < r->toSplit[d].getSize(); ++i) { Entity* e = r->toSplit[d][i]; @@ -53,11 +53,11 @@ void matchNewElements(Refine* r) packSplits(to,splits); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { Entity* e; PCU_COMM_UNPACK(e); diff --git a/ma/maMatchedCollapse.cc b/ma/maMatchedCollapse.cc index c54bfe75e..bb912eac5 100644 --- a/ma/maMatchedCollapse.cc +++ b/ma/maMatchedCollapse.cc @@ -79,13 +79,13 @@ void Rebuilds::match(apf::Sharing* sh) apf::CopyArray orig_matches; sh->getCopies(orig, orig_matches); for (unsigned j = 0; j < orig_matches.getSize(); ++j) { - PCU_ALWAYS_ASSERT(orig_matches[j].peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(orig_matches[j].peer == mesh->getPCU()->Self()); Entity* gen_match_j = 0; for (unsigned k = 0; k < v.size(); ++k) if (v[k].original == orig_matches[j].entity) gen_match_j = v[k].e; PCU_ALWAYS_ASSERT(gen_match_j); - mesh->addMatch(gen, PCU_Comm_Self(), gen_match_j); + mesh->addMatch(gen, mesh->getPCU()->Self(), gen_match_j); } } } @@ -124,7 +124,7 @@ void MatchedCollapse::setEdges() apf::CopyArray copies; sharing->getCopies(e, copies); APF_ITERATE(apf::CopyArray, copies, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == mesh->getPCU()->Self()); PCU_ALWAYS_ASSERT(it->entity != e); } collapses.setSize(copies.getSize() + 1); @@ -154,7 +154,7 @@ bool MatchedCollapse::checkTopo2() unsigned j; bool ok = false; for (j = 0; j < copies.getSize(); ++j) { - PCU_ALWAYS_ASSERT(copies[j].peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(copies[j].peer == mesh->getPCU()->Self()); if (copies[j].entity == collapses[i].vertToCollapse) { ok = true; break; diff --git a/ma/maMatchedSnapper.cc b/ma/maMatchedSnapper.cc index 87ce2c384..3c801f095 100644 --- a/ma/maMatchedSnapper.cc +++ b/ma/maMatchedSnapper.cc @@ -58,7 +58,7 @@ void MatchedSnapper::setVerts() apf::CopyArray copies; sharing->getCopies(v, copies); APF_ITERATE(apf::CopyArray, copies, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == adapter->mesh->getPCU()->Self()); PCU_ALWAYS_ASSERT(it->entity != v); } diff --git a/ma/maRefine.cc b/ma/maRefine.cc index df7f5c0ed..2e908f6ce 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -246,12 +246,12 @@ void splitElement(Refine* r, Entity* e) static void linkNewVerts(Refine* r) { - if (PCU_Comm_Peers()==1) + if (r->adapt->mesh->getPCU()->Peers()==1) return; struct { Entity* parent; Entity* vert; } message; Adapt* a = r->adapt; Mesh* m = a->mesh; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d=1; d < m->getDimension(); ++d) for (size_t i=0; i < r->newEntities[d].getSize(); ++i) { @@ -270,11 +270,11 @@ static void linkNewVerts(Refine* r) PCU_COMM_PACK(to,message); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { PCU_COMM_UNPACK(message); Entity* v = findSplitVert(r,message.parent); @@ -403,7 +403,7 @@ long markEdgesToSplit(Adapt* a) void processNewElements(Refine* r) { linkNewVerts(r); - if (PCU_Comm_Peers()>1) { + if (r->adapt->mesh->getPCU()->Peers()>1) { apf::stitchMesh(r->adapt->mesh); r->adapt->mesh->acceptChanges(); } diff --git a/ma/maSnap.cc b/ma/maSnap.cc index 195dca005..9f0f5a4cf 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -781,7 +781,7 @@ bool snapAllVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) SnapAll op(a, t, isSimple); applyOperator(a, &op); successCount += PCU_Add_Long(op.successCount); - return PCU_Or(op.didAnything); + return a->mesh->getPCU()->Or(op.didAnything); } class SnapMatched : public Operator @@ -832,7 +832,7 @@ bool snapMatchedVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) SnapMatched op(a, t, isSimple); applyOperator(a, &op); successCount += PCU_Add_Long(op.successCount); - return PCU_Or(op.didAnything); + return a->mesh->getPCU()->Or(op.didAnything); } long tagVertsToSnap(Adapt* a, Tag*& t) From 435172e0d6ac86e1e3c660970434836d81205179 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 23 Jan 2024 11:40:29 -0500 Subject: [PATCH 036/141] More fixing --- mds/apfMDS.cc | 2 +- parma/diffMC/parma_balancer.cc | 2 +- parma/diffMC/parma_centroids.cc | 10 ++++----- parma/diffMC/parma_dcpart.cc | 2 +- parma/diffMC/parma_dcpartFixer.cc | 2 +- parma/diffMC/parma_edgeEqVtxSelector.cc | 16 ++++++------- parma/diffMC/parma_elmBalancer.cc | 2 +- parma/diffMC/parma_elmLtVtxEdgeBalancer.cc | 6 ++--- parma/diffMC/parma_elmLtVtxEdgeSelector.cc | 16 ++++++------- parma/diffMC/parma_entWeights.cc | 12 +++++----- parma/diffMC/parma_ghost.cc | 12 +++++----- parma/diffMC/parma_ghostElement.cc | 6 ++--- parma/diffMC/parma_ghostMPAS.cc | 4 ++-- parma/diffMC/parma_ghostOwner.cc | 2 +- parma/diffMC/parma_graphDist.cc | 12 +++++----- parma/diffMC/parma_ltSelector.cc | 16 ++++++------- parma/diffMC/parma_step.cc | 4 ++-- parma/diffMC/parma_vtxBalancer.cc | 4 ++-- parma/diffMC/parma_vtxEdgeElmBalancer.cc | 8 +++---- parma/diffMC/parma_vtxElmBalancer.cc | 8 +++---- parma/diffMC/parma_vtxPtnWriter.cc | 16 ++++++------- parma/parma.cc | 26 +++++++++++----------- parma/rib/parma_mesh_rib.cc | 2 +- phasta/condense.cc | 2 +- phasta/phAdapt.cc | 8 +++---- phasta/phAxisymmetry.cc | 6 ++--- phasta/phBlock.cc | 2 +- phasta/phCook.cc | 24 ++++++++++---------- phasta/ph_convert.cc | 6 ++--- 29 files changed, 119 insertions(+), 119 deletions(-) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 4c082f4c7..568d50051 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -965,7 +965,7 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) vert_nums = mds_number_verts_bfs(m->mesh); } m->mesh = mds_reorder(m->mesh, 0, vert_nums); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"mesh reordered in %f seconds\n", PCU_Time()-t0); } diff --git a/parma/diffMC/parma_balancer.cc b/parma/diffMC/parma_balancer.cc index 742575b19..5e408fa28 100644 --- a/parma/diffMC/parma_balancer.cc +++ b/parma/diffMC/parma_balancer.cc @@ -33,7 +33,7 @@ namespace parma { } } void Balancer::balance(apf::MeshTag* wtag, double tolerance) { - if( 1 == PCU_Comm_Peers() ) return; + if( 1 == mesh->getPCU()->Peers() ) return; int step = 0; double t0 = PCU_Time(); while (runStep(wtag,tolerance) && step++ < maxStep); diff --git a/parma/diffMC/parma_centroids.cc b/parma/diffMC/parma_centroids.cc index 00aee0ec6..d7ba67a35 100644 --- a/parma/diffMC/parma_centroids.cc +++ b/parma/diffMC/parma_centroids.cc @@ -43,18 +43,18 @@ namespace parma { return x / weight; } - void Centroids::init(apf::Mesh*, Sides* s) { - PCU_Comm_Begin(); + void Centroids::init(apf::Mesh* m, Sides* s) { + m->getPCU()->Begin(); const Sides::Item* side; s->begin(); while( (side = s->iterate()) ) PCU_COMM_PACK(side->first, centroid); s->end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { apf::Vector3 otherCentroid; PCU_COMM_UNPACK(otherCentroid); - set(PCU_Comm_Sender(), otherCentroid); + set(m->getPCU()->Sender(), otherCentroid); } } } //end namespace diff --git a/parma/diffMC/parma_dcpart.cc b/parma/diffMC/parma_dcpart.cc index b7d3d1278..15005dad9 100644 --- a/parma/diffMC/parma_dcpart.cc +++ b/parma/diffMC/parma_dcpart.cc @@ -109,7 +109,7 @@ unsigned dcPart::numDisconnectedComps() { while( count != numElms ) { unsigned sz = walkPart(numDc); unsigned nbor = maxContactNeighbor(numDc); - if( nbor != self || PCU_Comm_Peers() == 1 ) { + if( nbor != self || m->getPCU()->Peers() == 1 ) { dcCompSz.push_back(sz); dcCompNbor.push_back(nbor); numDc++; diff --git a/parma/diffMC/parma_dcpartFixer.cc b/parma/diffMC/parma_dcpartFixer.cc index a778c2ff3..5766df221 100644 --- a/parma/diffMC/parma_dcpartFixer.cc +++ b/parma/diffMC/parma_dcpartFixer.cc @@ -87,7 +87,7 @@ class dcPartFixer::PartFixer : public dcPart { reset(); double t3 = PCU_Time(); m->migrate(plan); - if( ! PCU_Comm_Self() && vb) + if( ! m->getPCU()->Self() && vb) parmaCommons::status( "loop %d components %d seconds %.3f %.3f\n", loop, ndc, t3-t2, PCU_Time()-t3); diff --git a/parma/diffMC/parma_edgeEqVtxSelector.cc b/parma/diffMC/parma_edgeEqVtxSelector.cc index 048860b2b..31e8da6a6 100644 --- a/parma/diffMC/parma_edgeEqVtxSelector.cc +++ b/parma/diffMC/parma_edgeEqVtxSelector.cc @@ -138,16 +138,16 @@ namespace { typedef std::set MigrComm; - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, sendingVtx, s) PCU_COMM_PACK(s->first, s->second); - PCU_Comm_Send(); + mesh->getPCU()->Send(); MigrComm incoming; //map double w; - while (PCU_Comm_Listen()) { + while (mesh->getPCU()->Listen()) { PCU_COMM_UNPACK(w); - incoming.insert(Migr(PCU_Comm_Sender(),w)); + incoming.insert(Migr(mesh->getPCU()->Sender(),w)); } double selfW = parma::getWeight(mesh,wtag,0); @@ -165,15 +165,15 @@ namespace { totW += accept[in->first]; } - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, accept, a) PCU_COMM_PACK(a->first, a->second); - PCU_Comm_Send(); + mesh->getPCU()->Send(); parma::Mid* capacity = new parma::Mid; double outw; - while (PCU_Comm_Listen()) { + while (mesh->getPCU()->Listen()) { PCU_COMM_UNPACK(outw); - (*capacity)[PCU_Comm_Sender()] = outw; + (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; } diff --git a/parma/diffMC/parma_elmBalancer.cc b/parma/diffMC/parma_elmBalancer.cc index 757b2d25b..222ef59d9 100644 --- a/parma/diffMC/parma_elmBalancer.cc +++ b/parma/diffMC/parma_elmBalancer.cc @@ -33,7 +33,7 @@ namespace { monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("elmImb %f avgSides %f\n", maxElmImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc index 75e9eb81f..c23c9bfe1 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc @@ -24,7 +24,7 @@ namespace { : Balancer(m, f, v, "elements") { maxVtx = maxV; maxEdge = maxE; - if( !PCU_Comm_Self() && verbose ) { + if( !mesh->getPCU()->Self() && verbose ) { status("stepFactor %.3f\n", f); status("maxVtx %.3f\n", maxVtx); status("maxEdge %.3f\n", maxEdge); @@ -32,7 +32,7 @@ namespace { parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { @@ -53,7 +53,7 @@ namespace { double avgSides = parma::avgSharedSides(s); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("elmImb %f avgSides %f\n", maxElmImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc index 9fddb6d81..085b95804 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc @@ -137,7 +137,7 @@ namespace { } //send vtx and edge weight - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(PeerEntSet, peerVerts, pv) { const int dest = pv->first; double vw = weight(peerVerts[dest]); @@ -145,13 +145,13 @@ namespace { double ew = weight(peerEdges[dest]); PCU_COMM_PACK(dest, ew); } - PCU_Comm_Send(); + mesh->getPCU()->Send(); MigrComm incoming; double vw, ew; - while (PCU_Comm_Listen()) { + while (mesh->getPCU()->Listen()) { PCU_COMM_UNPACK(vw); PCU_COMM_UNPACK(ew); - incoming.insert(Migr(PCU_Comm_Sender(),vw,ew)); //Migr ctor does not exist + incoming.insert(Migr(mesh->getPCU()->Sender(),vw,ew)); //Migr ctor does not exist } Midd accept; @@ -182,18 +182,18 @@ namespace { totW[1] += accept[nbor].b; } - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(Midd, accept, acc) { PCU_COMM_PACK(acc->first, acc->second.a); PCU_COMM_PACK(acc->first, acc->second.b); } - PCU_Comm_Send(); + mesh->getPCU()->Send(); Midd* capacity = new Midd; dtwo outw; - while (PCU_Comm_Listen()) { + while (mesh->getPCU()->Listen()) { PCU_COMM_UNPACK(outw.a); PCU_COMM_UNPACK(outw.b); - (*capacity)[PCU_Comm_Sender()] = outw; + (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; } diff --git a/parma/diffMC/parma_entWeights.cc b/parma/diffMC/parma_entWeights.cc index 66c9dc2d7..b7305cc88 100644 --- a/parma/diffMC/parma_entWeights.cc +++ b/parma/diffMC/parma_entWeights.cc @@ -11,7 +11,7 @@ namespace parma { double getAvgWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return PCU_Add_Double(locW) / PCU_Comm_Peers(); + return PCU_Add_Double(locW) / m->getPCU()->Peers(); } double getWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { @@ -63,18 +63,18 @@ namespace parma { return entW; } - void EntWeights::init(apf::Mesh*, apf::MeshTag*, Sides* s) { - PCU_Comm_Begin(); + void EntWeights::init(apf::Mesh* m, apf::MeshTag*, Sides* s) { + m->getPCU()->Begin(); const Sides::Item* side; s->begin(); while( (side = s->iterate()) ) PCU_COMM_PACK(side->first, weight); s->end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { double otherWeight; PCU_COMM_UNPACK(otherWeight); - set(PCU_Comm_Sender(), otherWeight); + set(m->getPCU()->Sender(), otherWeight); } } Weights* makeEntWeights(apf::Mesh* m, apf::MeshTag* w, Sides* s, int dim) { diff --git a/parma/diffMC/parma_ghost.cc b/parma/diffMC/parma_ghost.cc index da42857e0..ee9f2b7f3 100644 --- a/parma/diffMC/parma_ghost.cc +++ b/parma/diffMC/parma_ghost.cc @@ -23,13 +23,13 @@ namespace { parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); double avgSides = parma::avgSharedSides(s); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); parma::GhostWeights* gw = @@ -40,7 +40,7 @@ namespace { double vtxImb, vtxAvg; parma::getImbalance(vtxW, vtxImb, vtxAvg); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f avg %.3f\n", vtxImb, vtxAvg); delete vtxW; @@ -73,13 +73,13 @@ namespace { parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); double avgSides = parma::avgSharedSides(s); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); parma::GhostWeights* gw = @@ -93,7 +93,7 @@ namespace { parma::getImbalance(elmW,elmImb,elmAvg); double edgeImb, edgeAvg; parma::getImbalance(edgeW, edgeImb, edgeAvg); - if( !PCU_Comm_Self() && verbose ) { + if( !mesh->getPCU()->Self() && verbose ) { status("elm imbalance %.3f avg %.3f\n", elmImb, elmAvg); status("edge imbalance %.3f avg %.3f\n", edgeImb, edgeAvg); } diff --git a/parma/diffMC/parma_ghostElement.cc b/parma/diffMC/parma_ghostElement.cc index 6d11a6cc5..dee8ef90d 100644 --- a/parma/diffMC/parma_ghostElement.cc +++ b/parma/diffMC/parma_ghostElement.cc @@ -23,13 +23,13 @@ namespace { parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); double avgSides = parma::avgSharedSides(s); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); parma::GhostWeights* gw = @@ -42,7 +42,7 @@ namespace { double faceImb, faceAvg, elmImb, elmAvg; parma::getImbalance(faceW, faceImb, faceAvg); parma::getImbalance(elmW, elmImb, elmAvg); - if( !PCU_Comm_Self() && verbose ) { + if( !mesh->getPCU()->Self() && verbose ) { status("face imbalance %.3f avg %.3f\n", faceImb, faceAvg); status("elm imbalance %.3f avg %.3f\n", elmImb, elmAvg); } diff --git a/parma/diffMC/parma_ghostMPAS.cc b/parma/diffMC/parma_ghostMPAS.cc index 5c2d6c806..d4840bdb5 100644 --- a/parma/diffMC/parma_ghostMPAS.cc +++ b/parma/diffMC/parma_ghostMPAS.cc @@ -20,7 +20,7 @@ namespace { parma::Sides* s = parma::makeElmBdrySides(mesh); sideTol = static_cast(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) lion_oprint(1, "sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { @@ -31,7 +31,7 @@ namespace { double avgSides = parma::avgSharedSides(s); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) lion_oprint(1, "avgSides %f\n", avgSides); parma::Weights* w = diff --git a/parma/diffMC/parma_ghostOwner.cc b/parma/diffMC/parma_ghostOwner.cc index 1fe51289a..853d6b757 100644 --- a/parma/diffMC/parma_ghostOwner.cc +++ b/parma/diffMC/parma_ghostOwner.cc @@ -11,6 +11,6 @@ namespace parma { } bool isOwned(apf::Mesh* m, apf::MeshEntity* v) { - return PCU_Comm_Self() == getOwner(m,v); + return m->getPCU()->Self() == getOwner(m,v); } } diff --git a/parma/diffMC/parma_graphDist.cc b/parma/diffMC/parma_graphDist.cc index f42861d44..89f2db853 100644 --- a/parma/diffMC/parma_graphDist.cc +++ b/parma/diffMC/parma_graphDist.cc @@ -291,7 +291,7 @@ namespace parma_ordering { if(start == TO_INT(m->count(0))) { if( i != 0 ) //if not the last component to order parmaCommons::status("%d all vertices visited comp %u of %u\n", - PCU_Comm_Self(), i, c.size()); + m->getPCU()->Self(), i, c.size()); break; } } @@ -346,7 +346,7 @@ namespace parma_ordering { long tot=PCU_Add_Long(TO_LONG(la)); int max=PCU_Max_Int(la); int min=PCU_Min_Int(la); - double avg = TO_DOUBLE(tot)/PCU_Comm_Peers(); + double avg = TO_DOUBLE(tot)/m->getPCU()->Peers(); if( !PCU_Comm_Self() ) parmaCommons::status("la min %d max %d avg %.3f\n", min, max, avg); PCU_ALWAYS_ASSERT(check == m->getTagChecksum(order,apf::Mesh::VERTEX)); @@ -368,11 +368,11 @@ namespace parma { PCU_Debug_Print("computeDistance\n"); dcComponents c = dcComponents(m); t = computeDistance(m,c); - if( PCU_Comm_Peers() > 1 && !c.numIso() ) + if( m->getPCU()->Peers() > 1 && !c.numIso() ) if( !hasDistance(m,t) ) { parmaCommons::error("rank %d comp %u iso %u ... " "some vertices don't have distance computed\n", - PCU_Comm_Self(), c.size(), c.numIso()); + m->getPCU()->Self(), c.size(), c.numIso()); PCU_ALWAYS_ASSERT(false); } unsigned* rmax = getMaxDist(m,c,t); @@ -390,11 +390,11 @@ apf::MeshTag* Parma_BfsReorder(apf::Mesh* m, int) { const unsigned checkIds = c.getIdChecksum(); apf::MeshTag* dist = computeDistance(m,c); const unsigned check = m->getTagChecksum(dist,apf::Mesh::VERTEX); - if( PCU_Comm_Peers() > 1 && !c.numIso() ) + if( m->getPCU()->Peers() > 1 && !c.numIso() ) if( !hasDistance(m,dist) ) { parmaCommons::error("rank %d comp %u iso %u ... " "some vertices don't have distance computed\n", - PCU_Comm_Self(), c.size(), c.numIso()); + m->getPCU()->Self(), c.size(), c.numIso()); PCU_ALWAYS_ASSERT(false); } parma_ordering::la(m); diff --git a/parma/diffMC/parma_ltSelector.cc b/parma/diffMC/parma_ltSelector.cc index 3e99068de..bb15cb458 100644 --- a/parma/diffMC/parma_ltSelector.cc +++ b/parma/diffMC/parma_ltSelector.cc @@ -109,16 +109,16 @@ namespace { typedef std::set MigrComm; - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, sendingEnts, s) PCU_COMM_PACK(s->first, s->second); - PCU_Comm_Send(); + mesh->getPCU()->Send(); MigrComm incoming; //map double w; - while (PCU_Comm_Listen()) { + while (mesh->getPCU()->Listen()) { PCU_COMM_UNPACK(w); - incoming.insert(Migr(PCU_Comm_Sender(),w)); + incoming.insert(Migr(mesh->getPCU()->Sender(),w)); } double selfW = parma::getWeight(mesh,wtag,primaryDim); @@ -136,15 +136,15 @@ namespace { totW += accept[in->first]; } - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, accept, a) PCU_COMM_PACK(a->first, a->second); - PCU_Comm_Send(); + mesh->getPCU()->Send(); parma::Mid* capacity = new parma::Mid; double outw; - while (PCU_Comm_Listen()) { + while (mesh->getPCU()->Listen()) { PCU_COMM_UNPACK(outw); - (*capacity)[PCU_Comm_Sender()] = outw; + (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; } diff --git a/parma/diffMC/parma_step.cc b/parma/diffMC/parma_step.cc index a30cc7d70..f23b7943e 100644 --- a/parma/diffMC/parma_step.cc +++ b/parma/diffMC/parma_step.cc @@ -30,7 +30,7 @@ namespace parma { bool Stepper::step(double maxImb, int verbosity) { double imb, avg; getImbalance(weights, imb, avg); - if ( !PCU_Comm_Self() && verbosity ) + if ( !m->getPCU()->Self() && verbosity ) status("%s imbalance %.3f avg %.3f\n", name, imb, avg); if ( stop->stop(imb,maxImb) ) return false; @@ -38,7 +38,7 @@ namespace parma { int planSz = PCU_Add_Int(plan->count()); const double t0 = PCU_Time(); m->migrate(plan); - if ( !PCU_Comm_Self() && verbosity ) + if ( !m->getPCU()->Self() && verbosity ) status("%d elements migrated in %f seconds\n", planSz, PCU_Time()-t0); if( verbosity > 1 ) Parma_PrintPtnStats(m, "endStep", (verbosity>2)); diff --git a/parma/diffMC/parma_vtxBalancer.cc b/parma/diffMC/parma_vtxBalancer.cc index 43edeae28..510681562 100644 --- a/parma/diffMC/parma_vtxBalancer.cc +++ b/parma/diffMC/parma_vtxBalancer.cc @@ -24,7 +24,7 @@ namespace { parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } @@ -39,7 +39,7 @@ namespace { double avgSides = parma::avgSharedSides(s); monitorUpdate(maxVtxImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtxImb %f avgSides %f\n", maxVtxImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_vtxEdgeElmBalancer.cc b/parma/diffMC/parma_vtxEdgeElmBalancer.cc index 83a1a3563..383340862 100644 --- a/parma/diffMC/parma_vtxEdgeElmBalancer.cc +++ b/parma/diffMC/parma_vtxEdgeElmBalancer.cc @@ -21,20 +21,20 @@ namespace { VtxEdgeBalancer(apf::Mesh* m, double f, double maxV, int v) : Balancer(m, f, v, "edges") { maxVtx = maxV; - if( !PCU_Comm_Self() && verbose ) { + if( !m->getPCU()->Self() && verbose ) { status("stepFactor %.3f\n", f); status("maxVtx %.3f\n", maxVtx); } parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { const double maxVtxImb = Parma_GetWeightedEntImbalance(mesh, wtag, 0); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f\n", maxVtxImb); const double maxEdgeImb = Parma_GetWeightedEntImbalance(mesh, wtag, 1); @@ -49,7 +49,7 @@ namespace { monitorUpdate(maxEdgeImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("edgeImb %f avgSides %f\n", maxEdgeImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_vtxElmBalancer.cc b/parma/diffMC/parma_vtxElmBalancer.cc index 14362e09f..11185b5ce 100644 --- a/parma/diffMC/parma_vtxElmBalancer.cc +++ b/parma/diffMC/parma_vtxElmBalancer.cc @@ -22,14 +22,14 @@ namespace { ElmLtVtx(apf::Mesh* m, double f, double maxV, int v) : Balancer(m, f, v, "elements") { maxVtx = maxV; - if( !PCU_Comm_Self() && verbose ) { + if( !m->getPCU()->Self() && verbose ) { status("stepFactor %.3f\n", f); status("maxVtx %.3f\n", maxVtx); } parma::Sides* s = parma::makeVtxSides(mesh); sideTol = TO_INT(parma::avgSharedSides(s)); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { @@ -37,7 +37,7 @@ namespace { Parma_GetWeightedEntImbalance(mesh, wtag, 0); const double maxElmImb = Parma_GetWeightedEntImbalance(mesh, wtag, mesh->getDimension()); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f\n", maxVtxImb); parma::Sides* s = parma::makeVtxSides(mesh); parma::Weights* vtxW = parma::makeEntWeights(mesh, wtag, s, 0); @@ -52,7 +52,7 @@ namespace { double avgSides = parma::avgSharedSides(s); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("elmImb %f avgSides %f\n", maxElmImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_vtxPtnWriter.cc b/parma/diffMC/parma_vtxPtnWriter.cc index 5e038081a..d6d03f850 100644 --- a/parma/diffMC/parma_vtxPtnWriter.cc +++ b/parma/diffMC/parma_vtxPtnWriter.cc @@ -22,10 +22,10 @@ namespace { public: Ptn(apf::Mesh* m) { const long totv = PCU_Add_Long(countOwned(m)); - c = pp = totv / PCU_Comm_Peers(); - f = pp * PCU_Comm_Self(); - const int remainder = totv % PCU_Comm_Peers(); - if( PCU_Comm_Self() == PCU_Comm_Peers()-1 ) + c = pp = totv / m->getPCU()->Peers(); + f = pp * m->getPCU()->Self(); + const int remainder = totv % m->getPCU()->Peers(); + if( m->getPCU()->Self() == m->getPCU()->Peers()-1 ) c += remainder; } int getWriter(int id) { @@ -48,7 +48,7 @@ namespace { apf::MeshEntity* vtx; apf::MeshIterator* itr = m->begin(0); int id = 0; - PCU_Comm_Begin(); + m->getPCU()->Begin(); while( (vtx = m->iterate(itr)) ) { if( parma::isOwned(m, vtx) ) { m->getIntTag(vtx, t, &id); @@ -56,13 +56,13 @@ namespace { } } m->end(itr); - PCU_Comm_Send(); - while( PCU_Comm_Receive() ) { + m->getPCU()->Send(); + while( m->getPCU()->Receive() ) { int id = 0; PCU_COMM_UNPACK(id); const int idx = id - p.first(); PCU_ALWAYS_ASSERT(idx >= 0 && idx < p.count()); - ptn[idx] = PCU_Comm_Sender(); + ptn[idx] = m->getPCU()->Sender(); } } diff --git a/parma/parma.cc b/parma/parma.cc index 56d57bec7..f3a64cd57 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -120,7 +120,7 @@ namespace { for(int d=0; d<=dim; d++) ss << entNames[d] << ' '; ss << "dc nb owned_bdry shared_bdry model_bdry shSidesToElm > " - << PCU_Comm_Self()+1 << ' '; + << m->getPCU()->Self()+1 << ' '; for(int d=0; d<=dim; d++) ss << m->count(d) << ' '; ss << m->count(m->getDimension()) << ' ' @@ -130,7 +130,7 @@ namespace { << surf/TO_DOUBLE(vol); std::string s = ss.str(); lion_eprint(1, "%s\n", s.c_str()); - PCU_Barrier(); + m->getPCU()->Barrier(); } void writeWeightedEntStats(apf::Mesh* m, apf::MeshTag* w, std::string key) { @@ -140,7 +140,7 @@ namespace { double totEnt[4] = {0,0,0,0}, avgEnt[4] = {0,0,0,0}; getWeightedStats(m->getDimension(), &weight, &totEnt, &minEnt, &maxEnt, &avgEnt); const char* orders[4] = {"vtx","edge","face","rgn"}; - if(!PCU_Comm_Self()) { + if(!m->getPCU()->Self()) { for( int d=0; d<=m->getDimension(); d++) status("%s weighted %s " "%.1f %.1f %.1f %.3f\n", @@ -171,7 +171,7 @@ void Parma_GetEntImbalance(apf::Mesh* mesh, double (*entImb)[4]) { PCU_Add_Doubles(tot, dims); PCU_Max_Doubles(*entImb, dims); for(size_t i=0; i < dims; i++) - (*entImb)[i] /= (tot[i]/PCU_Comm_Peers()); + (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) (*entImb)[i] = 1.0; } @@ -186,7 +186,7 @@ void Parma_GetWeightedEntImbalance(apf::Mesh* mesh, apf::MeshTag* w, PCU_Add_Doubles(tot, TO_SIZET(dims)); PCU_Max_Doubles(*entImb, TO_SIZET(dims)); for(size_t i=0; i < dims; i++) - (*entImb)[i] /= (tot[i]/PCU_Comm_Peers()); + (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) (*entImb)[i] = 1.0; } @@ -202,7 +202,7 @@ double Parma_GetWeightedEntImbalance(apf::Mesh* m, apf::MeshTag* w, m->end(it); double tot = PCU_Add_Double(sum); double max = PCU_Max_Double(sum); - return max/(tot/PCU_Comm_Peers()); + return max/(tot/m->getPCU()->Peers()); } void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, @@ -211,7 +211,7 @@ void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, getNeighborCounts(m,nborToShared); loc = TO_INT(nborToShared.size())-1; max = PCU_Max_Int(loc); - avg = TO_DOUBLE(PCU_Add_Int(loc)) / PCU_Comm_Peers(); + avg = TO_DOUBLE(PCU_Add_Int(loc)) / m->getPCU()->Peers(); numMaxParts = PCU_Add_Int( (loc==max) ); } @@ -225,7 +225,7 @@ void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { if( nbor->second == i+1 ) smallCnt[i]++; PCU_Add_Ints(smallCnt,small); - if( !PCU_Comm_Self() ) { + if( !m->getPCU()->Self() ) { std::stringstream ss; for(int i=0; igetPCU()->Peers(); } void Parma_ProcessDisconnectedParts(apf::Mesh* m) { @@ -331,7 +331,7 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, double surfToVol = surf/vol; double minSurfToVol = PCU_Min_Double(surfToVol); double maxSurfToVol = PCU_Max_Double(surfToVol); - double avgSurfToVol = PCU_Add_Double(surfToVol) / PCU_Comm_Peers(); + double avgSurfToVol = PCU_Add_Double(surfToVol) / m->getPCU()->Peers(); PCU_Debug_Print("%s sharedSidesToElements %.3f\n", key.c_str(), surfToVol); int empty = (m->count(m->getDimension()) == 0 ) ? 1 : 0; @@ -350,7 +350,7 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, PCU_Debug_Print("%d ", *p); PCU_Debug_Print("\n"); - if( 0 == PCU_Comm_Self() ) { + if( 0 == m->getPCU()->Self() ) { status("%s disconnected %d %.3f\n", key.c_str(), maxDc, avgDc); status("%s neighbors %d %.3f\n", @@ -365,7 +365,7 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, const int smallestSide = 10; Parma_WriteSmallNeighbors(m, smallestSide, key.c_str()); writeWeightedEntStats(m,w,key); - if( 0 == PCU_Comm_Self() ) { + if( 0 == m->getPCU()->Self() ) { status("%s owned bdry vtx " "%ld %d %d %.3f\n", key.c_str(), totV[0], maxV[0], minV[0], avgV[0]); @@ -412,7 +412,7 @@ int Parma_MisNumbering(apf::Mesh* m, int d) { int misNumber=-1; int iter=0; int misSize=0; - while( misSize != PCU_Comm_Peers() ) { + while( misSize != m->getPCU()->Peers() ) { if( mis(part, false, true) || 1 == part.net.size() ) { misNumber = iter; part.net.clear(); diff --git a/parma/rib/parma_mesh_rib.cc b/parma/rib/parma_mesh_rib.cc index 28b0c5450..ed0dff41f 100644 --- a/parma/rib/parma_mesh_rib.cc +++ b/parma/rib/parma_mesh_rib.cc @@ -76,7 +76,7 @@ class RibSplitter : public apf::Splitter plan->send(e, p + offset); } double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + if (!mesh->getPCU()->Self()) lion_oprint(1,"planned RIB factor %d in %f seconds\n", multiple, t1 - t0); } diff --git a/phasta/condense.cc b/phasta/condense.cc index e703059fd..14dfc90d3 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -57,7 +57,7 @@ int main(int argc, char** argv) { in.openfile_read = openfile_read; code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str()); chef::readAndAttachFields(in,code.mesh); - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); code.ctrl = in; Parma_ShrinkPartition(code.mesh, atoi(argv[2]), code); #ifdef HAVE_SIMMETRIX diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index 9ae8f783a..7f0fda1e3 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -96,19 +96,19 @@ struct AdaptCallback : public Parma_GroupCode static double getAveragePartDensity(apf::Mesh* m) { double nElements = m->count(m->getDimension()); nElements = PCU_Add_Double(nElements); - return nElements / PCU_Comm_Peers(); + return nElements / m->getPCU()->Peers(); } static int getShrinkFactor(apf::Mesh* m, double minPartDensity) { double partDensity = getAveragePartDensity(m); int factor = 1; while (partDensity < minPartDensity) { - if (factor >= PCU_Comm_Peers()) + if (factor >= m->getPCU()->Peers()) break; factor *= 2; partDensity *= 2; } - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() % factor == 0); + PCU_ALWAYS_ASSERT(m->getPCU()->Peers() % factor == 0); return factor; } @@ -123,7 +123,7 @@ static void warnAboutShrinking(int factor) { void adaptShrunken(apf::Mesh2* m, double minPartDensity, Parma_GroupCode& callback) { int factor = getShrinkFactor(m, minPartDensity); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_eprint(1,"adaptShrunken limit set to %f factor computed as %d\n", minPartDensity, factor); if (factor == 1) { callback.run(0); diff --git a/phasta/phAxisymmetry.cc b/phasta/phAxisymmetry.cc index 5aebe1dc8..d2834c2d5 100644 --- a/phasta/phAxisymmetry.cc +++ b/phasta/phAxisymmetry.cc @@ -111,7 +111,7 @@ apf::MeshTag* tagAngles(apf::Mesh* m, BCs& bcs, apf::MatchedSharing* ms) { apf::MeshTag* tag = m->createDoubleTag("ph_angle", 1); gmi_model* gm = m->getModel(); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* v; while ((v = m->iterate(it))) { @@ -132,8 +132,8 @@ apf::MeshTag* tagAngles(apf::Mesh* m, BCs& bcs, apf::MatchedSharing* ms) } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(v); int mdim, mtag; PCU_COMM_UNPACK(mdim); diff --git a/phasta/phBlock.cc b/phasta/phBlock.cc index 573532e6d..6c5ffe03f 100644 --- a/phasta/phBlock.cc +++ b/phasta/phBlock.cc @@ -202,7 +202,7 @@ void getInterfaceBlocks(apf::Mesh* m, BCs& bcs, BlocksInterface& b) if (dgCopies.getSize() != 1) continue; apf::MeshEntity* e0 = m->getUpward(face, 0); - PCU_ALWAYS_ASSERT(dgCopies[0].peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(dgCopies[0].peer == m->getPCU()->Self()); apf::MeshEntity* e1 = m->getUpward(dgCopies[0].entity, 0); /* in order to avoid repetition of elements */ if (e0 > e1) diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 983570d43..d35a3fada 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -149,13 +149,13 @@ namespace ph { void checkBalance(apf::Mesh2* m, ph::Input& in) { /* check if balancing was requested */ Parma_PrintPtnStats(m, "postSplit", false); - if (in.prePhastaBalanceMethod != "none" && PCU_Comm_Peers() > 1) + if (in.prePhastaBalanceMethod != "none" && m->getPCU()->Peers() > 1) ph::balance(in,m); } void checkReorder(apf::Mesh2* m, ph::Input& in, int numMasters) { /* check if the mesh changed at all */ - if ( (PCU_Comm_Peers()!=numMasters) || + if ( (m->getPCU()->Peers()!=numMasters) || in.splitFactor > 1 || in.adaptFlag || in.prePhastaBalanceMethod != "none" || @@ -166,7 +166,7 @@ namespace ph { print_stats("malloc used before Bfs", PCU_GetMem()); - if (in.isReorder && PCU_Comm_Peers() > 1) + if (in.isReorder && m->getPCU()->Peers() > 1) order = Parma_BfsReorder(m); print_stats("malloc used before reorder", PCU_GetMem()); @@ -185,10 +185,10 @@ namespace ph { void preprocess(apf::Mesh2* m, Input& in, Output& out, BCs& bcs) { phastaio_initStats(); - if(PCU_Comm_Peers() > 1) + if(m->getPCU()->Peers() > 1) ph::migrateInterfaceItr(m, bcs); if (in.simmetrixMesh == 0) - ph::checkReorder(m,in,PCU_Comm_Peers()); + ph::checkReorder(m,in,m->getPCU()->Peers()); if (in.adaptFlag) ph::goToStepDir(in.timeStepNumber,in.ramdisk); std::string path = ph::setupOutputDir(in.ramdisk); @@ -199,7 +199,7 @@ namespace ph { ph::exitFilteredMatching(m); // a path is not needed for inmem if ( in.writeRestartFiles ) { - if(!PCU_Comm_Self()) lion_oprint(1,"write file-based restart file\n"); + if(!m->getPCU()->Self()) lion_oprint(1,"write file-based restart file\n"); // store the value of the function pointer FILE* (*fn)(Output& out, const char* path) = out.openfile_write; // set function pointer for file writing @@ -214,7 +214,7 @@ namespace ph { if ( ! in.outMeshFileName.empty() ) m->writeNative(in.outMeshFileName.c_str()); if ( in.writeGeomBCFiles ) { - if(!PCU_Comm_Self()) lion_oprint(1,"write additional geomBC file for visualization\n"); + if(!m->getPCU()->Self()) lion_oprint(1,"write additional geomBC file for visualization\n"); // store the value of the function pointer FILE* (*fn)(Output& out, const char* path) = out.openfile_write; // set function pointer for file writing @@ -224,7 +224,7 @@ namespace ph { out.openfile_write = fn; } ph::writeGeomBC(out, subDirPath); //write geombc - if(!PCU_Comm_Self()) + if(!m->getPCU()->Self()) ph::writeAuxiliaryFiles(path, in.timeStepNumber); m->verify(); #ifdef HAVE_SIMMETRIX @@ -272,11 +272,11 @@ namespace chef { shrinkFactor=-1*in.splitFactor; in.splitFactor=1; // this is used in to set readers so if shrinking need to read all } - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() % in.splitFactor == 0); + PCU_ALWAYS_ASSERT(m->getPCU()->Peers() % in.splitFactor == 0); apf::Migration* plan = 0; ph::BCs bcs; loadCommon(in, bcs, g); - const int worldRank = PCU_Comm_Self(); + const int worldRank = m->getPCU()->Self(); MPI_Comm comm = PCU_Get_Comm(); switchToMasters(in.splitFactor); if ((worldRank % in.splitFactor) == 0) @@ -286,7 +286,7 @@ namespace chef { m = repeatMdsMesh(m, g, plan, in.splitFactor); if (in.simmetrixMesh == 0 && shrinkFactor > 1){ GroupCode code; - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + apf::Unmodulo outMap(m->getPCU()->Self(), m->getPCU()->Peers()); code.mesh=m; code.input=∈ code.boundary=&bcs; @@ -365,7 +365,7 @@ namespace chef { } void balanceAndReorder(ph::Input& ctrl, apf::Mesh2* m) { - ph::balanceAndReorder(m,ctrl,PCU_Comm_Peers()); + ph::balanceAndReorder(m,ctrl,m->getPCU()->Peers()); } void balance(ph::Input& ctrl, apf::Mesh2* m) { diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 52e22a6af..3c5d7c4e5 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -154,14 +154,14 @@ static void fixCoords(apf::Mesh2* m) } } m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); double max_x_diff = 0; double max_p_diff = 0; apf::Vector3 max_x_diff_point; apf::Vector3 max_p_diff_point; int x_diffs = 0; int p_diffs = 0; - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { apf::Vector3 ox, op; PCU_COMM_UNPACK(e); PCU_COMM_UNPACK(ox); @@ -257,7 +257,7 @@ int main(int argc, char** argv) apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh); double t3 = PCU_Time(); - if(!PCU_Comm_Self()) + if(!mesh->getPCU()->Self()) lion_eprint(1, "created the apf_mds mesh in %f seconds\n", t3-t2); apf::printStats(mesh); From 3f40db9ff25664242499fc0dbadacd4179178a6e Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 24 Jan 2024 16:39:01 -0500 Subject: [PATCH 037/141] ph and pumi changes --- apf/apfCavityOp.cc | 2 +- pcu/PCUObj.cc | 2 + pcu/PCUObj.h | 13 +++++ phasta/phAdapt.cc | 2 +- phasta/phCook.cc | 6 +- phasta/phFilterMatching.cc | 8 +-- phasta/phGeomBC.cc | 8 +-- phasta/phGrowthCurves.cc | 10 ++-- phasta/phGrowthCurves_empty.cc | 2 +- phasta/phInterfaceCutter.cc | 8 +-- phasta/phLinks.cc | 18 +++--- phasta/phOutput.cc | 78 ++++++++++++------------- phasta/phPartition.cc | 6 +- phasta/phRestart.cc | 20 +++---- pumi/pumi_field.cc | 14 ++--- pumi/pumi_geom.cc | 4 +- pumi/pumi_ghost.cc | 72 +++++++++++------------ pumi/pumi_mesh.cc | 102 ++++++++++++++++----------------- pumi/pumi_numbering.cc | 14 ++--- 19 files changed, 202 insertions(+), 187 deletions(-) diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index e89aa9c66..a1600b2c2 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -156,7 +156,7 @@ bool CavityOp::sendPullRequests(std::vector& received) { int remotePart = rit->peer; MeshEntity* remoteEntity = rit->entity; - PCU_COMM_PACK(remotePart,remoteEntity); } + mesh->getPCU()->Pack(remotePart,remoteEntity); } } requests.clear(); mesh->getPCU()->Send(); diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index 40fe39ee5..0a92f5c6f 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -27,6 +27,7 @@ int PCU::Pack(int to_rank, const void *data, size_t size) noexcept { memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); return PCU_SUCCESS; } + int PCU::Send() noexcept { pcu_msg_send(mpi_, msg_); return PCU_SUCCESS; @@ -59,6 +60,7 @@ int PCU::Unpack(void *data, size_t size) noexcept { memcpy(data, pcu_msg_unpack(msg_, size), size); return PCU_SUCCESS; } + int PCU::Write(int to_rank, const void *data, size_t size) noexcept { if ((to_rank < 0) || (to_rank >= Peers())) reel_fail("Invalid rank in Comm_Write"); diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index aea6f263c..23a608ffd 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -29,12 +29,25 @@ class PCU { /*recommended message passing API*/ void Begin() noexcept; int Pack(int to_rank, const void *data, size_t size) noexcept; + template int Pack(int to_rank, T& data) noexcept { + return Pack(to_rank, &(data), sizeof(data)); + } + template int Pack(int to_rank, T* data) noexcept { + return Pack(to_rank, &(data), sizeof(data)); + } + int Send() noexcept; bool Receive() noexcept; bool Listen() noexcept; int Sender() noexcept; bool Unpacked() noexcept; int Unpack(void *data, size_t size) noexcept; + template int Unpack(T& data) noexcept { + return Unpack(&(data), sizeof(data)); + } + template int Unpack(T* data) noexcept { + return Unpack(&(data), sizeof(data)); + } /*IPComMan replacement API*/ int Write(int to_rank, const void *data, size_t size) noexcept; bool Read(int *from_rank, void **data, size_t *size) noexcept; diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index 7f0fda1e3..05ee264be 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -95,7 +95,7 @@ struct AdaptCallback : public Parma_GroupCode static double getAveragePartDensity(apf::Mesh* m) { double nElements = m->count(m->getDimension()); - nElements = PCU_Add_Double(nElements); + nElements = m->getPCU()->Add(nElements); return nElements / m->getPCU()->Peers(); } diff --git a/phasta/phCook.cc b/phasta/phCook.cc index d35a3fada..6c049f4e9 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -164,16 +164,16 @@ namespace ph { { apf::MeshTag* order = NULL; - print_stats("malloc used before Bfs", PCU_GetMem()); + print_stats("malloc used before Bfs", pcu::GetMem()); if (in.isReorder && m->getPCU()->Peers() > 1) order = Parma_BfsReorder(m); - print_stats("malloc used before reorder", PCU_GetMem()); + print_stats("malloc used before reorder", pcu::GetMem()); apf::reorderMdsMesh(m,order); - print_stats("malloc used after reorder", PCU_GetMem()); + print_stats("malloc used after reorder", pcu::GetMem()); } } diff --git a/phasta/phFilterMatching.cc b/phasta/phFilterMatching.cc index 0aa5f4170..dedaeadfa 100644 --- a/phasta/phFilterMatching.cc +++ b/phasta/phFilterMatching.cc @@ -225,7 +225,7 @@ void filterMatching(apf::Mesh2* m, ModelMatching& mm, int dim) { gmi_model* gm; gm = m->getModel(); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(dim); apf::MeshEntity* e; int gd, gt; @@ -246,8 +246,8 @@ void filterMatching(apf::Mesh2* m, ModelMatching& mm, int dim) m->clearMatches(e); } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(e); apf::MeshEntity* oe; PCU_COMM_UNPACK(oe); @@ -259,7 +259,7 @@ void filterMatching(apf::Mesh2* m, ModelMatching& mm, int dim) ModelSet& ms = mm[ge]; gmi_ent* oge = gmi_find(gm, gd, gt); if (oge == ge || ms.count(oge)) - m->addMatch(e, PCU_Comm_Sender(), oe); + m->addMatch(e, m->getPCU()->Sender(), oe); } checkFilteredMatching(m, mm, dim); } diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc index c61b4064c..4f88d1f00 100644 --- a/phasta/phGeomBC.cc +++ b/phasta/phGeomBC.cc @@ -294,7 +294,7 @@ static void writeSpanwiseAvgArrays(Output& o, FILE* f) void writeGeomBC(Output& o, std::string path, int timestep) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); apf::Mesh* m = o.mesh; std::stringstream tss; std::string timestep_or_dat; @@ -354,7 +354,7 @@ void writeGeomBC(Output& o, std::string path, int timestep) params[1] = 3; ph_write_doubles(f, "co-ordinates", o.arrays.coordinates, params[0] * params[1], 2, params); - writeInt(f, "number of processors", PCU_Comm_Peers()); + writeInt(f, "number of processors", m->getPCU()->Peers()); writeInt(f, "size of ilwork array", o.nlwork); if (o.nlwork) writeInts(f, "ilwork ", o.arrays.ilwork, o.nlwork); @@ -373,8 +373,8 @@ void writeGeomBC(Output& o, std::string path, int timestep) writeGrowthCurves(o, f); writeSpanwiseAvgArrays(o, f); PHASTAIO_CLOSETIME(fclose(f);) - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"geombc file written in %f seconds\n", t1 - t0); } diff --git a/phasta/phGrowthCurves.cc b/phasta/phGrowthCurves.cc index d8bef4e65..c3cc0c343 100644 --- a/phasta/phGrowthCurves.cc +++ b/phasta/phGrowthCurves.cc @@ -236,12 +236,12 @@ void getGrowthCurves(Output& o) o.arrays.igclv[i] = me; } - lion_oprint(1,"%s: rank %d, ngc, nv: %d, %d\n", __func__, PCU_Comm_Self(), ngc, nv); + lion_oprint(1,"%s: rank %d, ngc, nv: %d, %d\n", __func__, o.mesh->getPCU()->Self(), ngc, nv); - PCU_Add_Ints(&ngc,sizeof(ngc)); - PCU_Add_Ints(&nv,sizeof(nv)); + o.mesh->getPCU()->Add(&ngc,sizeof(ngc)); + o.mesh->getPCU()->Add(&nv,sizeof(nv)); - if(PCU_Comm_Self() == 0) + if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"%s: total ngc, nv: %d, %d\n", __func__, ngc, nv); PList_delete(gEdges); @@ -261,7 +261,7 @@ void getGrowthCurves(Output& o) Sim_logOff(); } else { - if(PCU_Comm_Self() == 0) + if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"%s: warning! not implemented for MDS mesh\n",__func__); } return; diff --git a/phasta/phGrowthCurves_empty.cc b/phasta/phGrowthCurves_empty.cc index c37eb6874..5cfafa07f 100644 --- a/phasta/phGrowthCurves_empty.cc +++ b/phasta/phGrowthCurves_empty.cc @@ -6,7 +6,7 @@ namespace ph { void getGrowthCurves(Output& o) { o.nGrowthCurves = 0; o.nLayeredMeshVertices = 0; - if(PCU_Comm_Self() == 0) + if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"warning! \'%s\' requires the Simmetrix SimAdvMeshing library\n",__func__); return; } diff --git a/phasta/phInterfaceCutter.cc b/phasta/phInterfaceCutter.cc index 90fe0d2dc..55547b963 100644 --- a/phasta/phInterfaceCutter.cc +++ b/phasta/phInterfaceCutter.cc @@ -262,18 +262,18 @@ int migrateInterface(apf::Mesh2*& m, ph::BCs& bcs) { int remoteResidence = -1; for (size_t j = 0; j != dgCopies.getSize(); ++j) { - if (dgCopies[j].peer != PCU_Comm_Self()) + if (dgCopies[j].peer != m->getPCU()->Self()) remoteResidence = dgCopies[j].peer; } - if (remoteResidence > PCU_Comm_Self()) + if (remoteResidence > m->getPCU()->Self()) plan->send(e,remoteResidence); } m->end(it); - lion_oprint(1,"proc-%d: number of migrating elements: %d\n",PCU_Comm_Self(),plan->count()); + lion_oprint(1,"proc-%d: number of migrating elements: %d\n",m->getPCU()->Self(),plan->count()); int totalPlan = plan->count(); - totalPlan = PCU_Add_Int(totalPlan); + totalPlan = m->getPCU()->Add(totalPlan); m->migrate(plan); return totalPlan; diff --git a/phasta/phLinks.cc b/phasta/phLinks.cc index 2b64aa52e..838018bc1 100644 --- a/phasta/phLinks.cc +++ b/phasta/phLinks.cc @@ -54,7 +54,7 @@ struct PhastaSharing : public apf::Sharing { if ( ! mesh->hasMatching()) return; /* filter out matches which are on the same part as the global master */ - int self = PCU_Comm_Self(); + int self = mesh->getPCU()->Self(); size_t i = 0; for (size_t j = 0; j < copies.getSize(); ++j) if (copies[j].peer != self) @@ -99,7 +99,7 @@ struct PhastaSharing : public apf::Sharing { void getLinks(apf::Mesh* m, int dim, Links& links, BCs& bcs) { PhastaSharing shr(m); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(dim); apf::MeshEntity* v; while ((v = m->iterate(it))) { @@ -118,16 +118,16 @@ void getLinks(apf::Mesh* m, int dim, Links& links, BCs& bcs) of the same master in the outgoing links array to a part that contains multiple copies of it. */ links[LinkKey(1, remotes[i].peer)].push_back(v); - PCU_COMM_PACK(remotes[i].peer, remotes[i].entity); + m->getPCU()->Pack(remotes[i].peer, remotes[i].entity); } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int peer = PCU_Comm_Sender(); - while (!PCU_Comm_Unpacked()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { + int peer = m->getPCU()->Sender(); + while (!m->getPCU()->Unpacked()) { apf::MeshEntity* v; - PCU_COMM_UNPACK(v); + m->getPCU()->Unpack(v); links[LinkKey(0, peer)].push_back(v); } } @@ -217,7 +217,7 @@ static apf::MeshEntity* getOtherElem(apf::Mesh* m, apf::MeshEntity* elem, return 0; apf::Matches matches; m->getMatches(face, matches); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); for (size_t i = 0; i < matches.getSize(); ++i) if (matches[i].peer == self) return m->getUpward(matches[i].entity, 0); diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc index d4b71028b..e4606acec 100644 --- a/phasta/phOutput.cc +++ b/phasta/phOutput.cc @@ -32,23 +32,23 @@ static void getCounts(Output& o) static void checkLoadBalance(Output& o) { - long sumOwnedNodes = PCU_Add_Long(o.nOwnedNodes); - long sumAllNodes = PCU_Add_Long(o.nOverlapNodes); - double avgNodes = static_cast(sumAllNodes) / PCU_Comm_Peers(); + long sumOwnedNodes = o.mesh->getPCU()->Add(o.nOwnedNodes); + long sumAllNodes = o.mesh->getPCU()->Add(o.nOverlapNodes); + double avgNodes = static_cast(sumAllNodes) / o.mesh->getPCU()->Peers(); double vlbratio = o.nOverlapNodes / avgNodes; - double vlbratio_max = PCU_Max_Double(vlbratio); - if (!PCU_Comm_Self()) + double vlbratio_max = o.mesh->getPCU()->Max(vlbratio); + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"max vertex load imbalance of partitioned mesh = %f\n", vlbratio_max); - if (!PCU_Comm_Self()) + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"ratio of sum of all vertices to sum of owned vertices = %f\n", sumAllNodes / (double) sumOwnedNodes); int dim = o.mesh->getDimension(); int numElms = o.mesh->count(dim); - long sumElms = PCU_Add_Long(numElms); - double avgElms = static_cast(sumElms) / PCU_Comm_Peers(); + long sumElms = o.mesh->getPCU()->Add(numElms); + double avgElms = static_cast(sumElms) / o.mesh->getPCU()->Peers(); double elbratio = numElms / avgElms; - double elbratio_max = PCU_Max_Double(elbratio); - if (!PCU_Comm_Self()) + double elbratio_max = o.mesh->getPCU()->Max(elbratio); + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"max region (3D) or face (2D) load imbalance of partitioned mesh = %f\n", elbratio_max); } @@ -115,8 +115,8 @@ static void getGlobal(Output& o) { apf::Mesh* m = o.mesh; int n = m->count(0); - int self = PCU_Comm_Self(); - int peers = PCU_Comm_Peers(); + int self = m->getPCU()->Self(); + int peers = m->getPCU()->Peers(); int id = self + 1; o.arrays.globalNodeNumbers = new int[n]; for (int i = 0; i < n; ++i) { @@ -299,7 +299,7 @@ static void getBoundary(Output& o, BCs& bcs, apf::Numbering* n) } static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { - PCU_Comm_Begin(); + o.mesh->getPCU()->Begin(); apf::Mesh* m = o.mesh; gmi_model* gm = m->getModel(); int rbID = 0; // id - set by user @@ -343,8 +343,8 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { rit = rbIDmap.find(rbID); if(rit == rbIDmap.end()) { rbIDmap[rbID] = rbMT; - PCU_Comm_Pack(0, &rbID, sizeof(int)); - PCU_Comm_Pack(0, &rbMT, sizeof(int)); + m->getPCU()->Pack(0, &rbID, sizeof(int)); + m->getPCU()->Pack(0, &rbMT, sizeof(int)); } int vID = apf::getNumber(n, e, 0, 0); if(f[vID] > -1 && f[vID] != rbID) { @@ -359,22 +359,22 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { } // end if rigid body attribute exists // master receives and form the complete set - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int rrbID = 0; int rrbMT = 0; - PCU_Comm_Unpack(&rrbID, sizeof(int)); - PCU_Comm_Unpack(&rrbMT, sizeof(int)); + m->getPCU()->Unpack(&rrbID, sizeof(int)); + m->getPCU()->Unpack(&rrbMT, sizeof(int)); rit = rbIDmap.find(rrbID); if(rit == rbIDmap.end()) rbIDmap[rrbID] = rrbMT; } - int rbIDs_size = PCU_Max_Int(rbIDmap.size()); + int rbIDs_size = m->getPCU()->Max(rbIDmap.size()); int* rbIDs = new int[rbIDs_size](); int* rbMTs = new int[rbIDs_size](); - if (!PCU_Comm_Self()) { + if (!m->getPCU()->Self()) { int count = 0; for (rit=rbIDmap.begin(); rit!=rbIDmap.end(); rit++) { rbIDs[count] = rit->first; @@ -385,8 +385,8 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { } // allreduce the set - PCU_Max_Ints(rbIDs, rbIDs_size); - PCU_Max_Ints(rbMTs, rbIDs_size); + m->getPCU()->Max(rbIDs, rbIDs_size); + m->getPCU()->Max(rbMTs, rbIDs_size); // attach data o.numRigidBody = rbIDs_size; @@ -434,7 +434,7 @@ bool checkInterface(Output& o, BCs& bcs) { m->end(it); PCU_ALWAYS_ASSERT(aID!=bID); //assert different material ID on two sides PCU_ALWAYS_ASSERT(a==b); //assert same number of faces on each side - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) lion_oprint(1,"Checked! Same number of faces on each side of interface.\n"); return true; } @@ -739,7 +739,7 @@ static void getGCEssentialBCs(Output& o, apf::Numbering* n) apf::Mesh* m = o.mesh; if(!in.ensa_melas_dof) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); int nec = countEssentialBCs(in); int& ei = o.nEssentialBCNodes; @@ -793,9 +793,9 @@ static void getGCEssentialBCs(Output& o, apf::Numbering* n) if(j == igcnv - 1 && m->isShared(vent)){ m->getRemotes(vent, remotes); APF_ITERATE(apf::Copies, remotes, rit) { - PCU_COMM_PACK(rit->first, rit->second); - PCU_Comm_Pack(rit->first, &ibc, sizeof(int)); - PCU_Comm_Pack(rit->first, &(bbc[0]), nec*sizeof(double)); + m->getPCU()->Pack(rit->first, rit->second); + m->getPCU()->Pack(rit->first, &ibc, sizeof(int)); + m->getPCU()->Pack(rit->first, &(bbc[0]), nec*sizeof(double)); } } } // end loop over nodes on a growth curve @@ -803,14 +803,14 @@ static void getGCEssentialBCs(Output& o, apf::Numbering* n) } // receive top most node - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { apf::MeshEntity* rvent; - PCU_COMM_UNPACK(rvent); + m->getPCU()->Unpack(rvent); int ribc = 0; - PCU_Comm_Unpack(&ribc, sizeof(int)); + m->getPCU()->Unpack(&ribc, sizeof(int)); double* rbc = new double[nec]; - PCU_Comm_Unpack(&(rbc[0]), nec*sizeof(double)); + m->getPCU()->Unpack(&(rbc[0]), nec*sizeof(double)); vID = apf::getNumber(n, rvent, 0, 0); if(o.arrays.nbc[vID] <= 0){ o.arrays.nbc[vID] = ei + 1; @@ -841,7 +841,7 @@ static void getInitialConditions(BCs& bcs, Output& o) { Input& in = *o.in; if (in.solutionMigration) { - if (!PCU_Comm_Self()) + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"All attribute-based initial conditions, " "if any, " "are ignored due to request for SolutionMigration\n"); @@ -967,10 +967,10 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) { o.arrays.ifather = new int[nnodes]; //initialize ifath apf::MeshTag* t = m->findTag("fathers2D"); if (t==NULL) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"Did not find tag fathers2D\n"); } else if (t != NULL) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"Found tag fathers2D\n"); } int tagNum; @@ -1073,7 +1073,7 @@ Output::~Output() void generateOutput(Input& in, BCs& bcs, apf::Mesh* mesh, Output& o) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); o.in = ∈ o.mesh = mesh; getCounts(o); @@ -1107,8 +1107,8 @@ void generateOutput(Input& in, BCs& bcs, apf::Mesh* mesh, Output& o) apf::destroyNumbering(rn); if (in.initBubbles) initBubbles(o.mesh, in); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"generated output structs in %f seconds\n",t1 - t0); } diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc index 8546c4188..7cd11eeec 100644 --- a/phasta/phPartition.cc +++ b/phasta/phPartition.cc @@ -94,7 +94,7 @@ bool isMixed(apf::Mesh2* m) { break; } m->end(it); - return PCU_Max_Int(mixed); + return m->getPCU()->Max(mixed); } void clearTags(apf::Mesh* m, apf::MeshTag* t) { @@ -144,7 +144,7 @@ void parmaTet(Input& in, apf::Mesh2* m, bool runGap) { neighborReduction(m,weights,verbose,fineStats); double vtxImb = Parma_GetWeightedEntImbalance(m, weights, 0); if( vtxImb <= in.vertexImbalance ) { - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) lion_oprint(1, "STATUS vtx imbalance target %.3f reached\n", in.vertexImbalance); break; @@ -206,7 +206,7 @@ void simmetrixBalance(apf::Mesh2* m) // current total num parts in pmesh cannot be more than requested int currentTotalNumParts = PM_totalNumParts(pmesh); if (currentTotalNumParts > totalNumParts) { - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) lion_eprint(1, "Error: cannot reduce number of partitions %d->%d\n", currentTotalNumParts, totalNumParts); totalNumParts = currentTotalNumParts; diff --git a/phasta/phRestart.cc b/phasta/phRestart.cc index 2135f2616..3de57dec7 100644 --- a/phasta/phRestart.cc +++ b/phasta/phRestart.cc @@ -29,7 +29,7 @@ apf::Field* extractField(apf::Mesh* m, bool simField) { apf::Field* f = m->findField(packedFieldname); - if(!f && PCU_Comm_Self() == 0) + if(!f && m->getPCU()->Self() == 0) lion_eprint(1, "No packed field \"%s\"", packedFieldname); PCU_ALWAYS_ASSERT(f); apf::Field* rf = m->findField(requestFieldname); @@ -321,7 +321,7 @@ static bool isNodalField(const char* fieldname, int nnodes, apf::Mesh* m) for (int i = 0; i < known_rand_field_count; ++i) if (!strcmp(fieldname, known_rand_fields[i])) return false; - if( !PCU_Comm_Self() ) { + if( !m->getPCU()->Self() ) { lion_eprint(1, "unknown restart field name \"%s\"\n", fieldname); lion_eprint(1, "please add \"%s\" to isNodalField above line %d of %s\n", fieldname, __LINE__, __FILE__); @@ -363,7 +363,7 @@ int readAndAttachField( if ( std::string(hname) == std::string("solution") ) out_size = in.ensa_dof; if (m->findField(hname)) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_eprint(1, "field \"%s\" already attached to the mesh, " "ignoring request to re-attach...\n", hname); } else { @@ -443,7 +443,7 @@ static double* buildMappingPartId(apf::Mesh* m) int n = m->count(0); /* malloc instead of new[] for consistency with ph_read_field */ double* data = (double*)malloc(sizeof(double) * n); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); for (int i = 0; i < n; ++i) data[i] = self; return data; @@ -468,7 +468,7 @@ static std::string buildRestartFileName(std::string prefix, int step) void readAndAttachFields(Input& in, apf::Mesh* m) { phastaio_initStats(); - double t0 = PCU_Time(); + double t0 = pcu::Time(); setupInputSubdir(in.restartFileName); std::string filename = buildRestartFileName(in.restartFileName, in.timeStepNumber); phastaio_setfile(RESTART_READ); @@ -481,8 +481,8 @@ void readAndAttachFields(Input& in, apf::Mesh* m) { /* stops when ph_read_field returns 0 */ while( readAndAttachField(in,f,m,swap) ) {} PHASTAIO_CLOSETIME(fclose(f);) - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"fields read and attached in %f seconds\n", t1 - t0); if(in.printIOtime) phastaio_printStats(); } @@ -517,7 +517,7 @@ void attachZeroSolution(Input& in, apf::Mesh* m) void detachAndWriteSolution(Input& in, Output& out, apf::Mesh* m, std::string path) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); path += buildRestartFileName("restart", in.timeStepNumber); phastaio_setfile(RESTART_WRITE); FILE* f = out.openfile_write(out, path.c_str()); @@ -564,8 +564,8 @@ void detachAndWriteSolution(Input& in, Output& out, apf::Mesh* m, std::string pa while(m->countFields()) apf::destroyField( m->getField(0) ); PHASTAIO_CLOSETIME(fclose(f);) - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"solution written in %f seconds\n", t1 - t0); } diff --git a/pumi/pumi_field.cc b/pumi/pumi_field.cc index 74fb55574..6ca417eb4 100644 --- a/pumi/pumi_field.cc +++ b/pumi/pumi_field.cc @@ -389,7 +389,7 @@ static void sendFieldData(pMesh m, pMeshEnt e, pField f, int nf, pOwnership shr) double *s_data = (double*)((char*)msg_send+sizeof(pMeshEnt)+sizeof(int)); for (int pos=0; posgetPCU()->Write(to, (void*)msg_send, msg_size); free(msg_send); } @@ -408,7 +408,7 @@ static void sendFieldData(pMesh m, pMeshEnt e, pField f, int nf, pOwnership shr) double *s_data = (double*)((char*)msg_send+sizeof(pMeshEnt)+sizeof(int)); for (int pos=0; posgetPCU()->Write(to, (void*)msg_send, msg_size); free(msg_send); } } //if (m->isGhosted(e)) @@ -484,26 +484,26 @@ void pumi_field_verify(pMesh m, pField f, pOwnership shr) if (!static_cast(fields[nf])->getShape()->hasNodesIn(d)) continue; - PCU_Comm_Begin(); + m->getPCU()->Begin(); pMeshIter it = m->begin(d); pMeshEnt e; while ((e = m->iterate(it))) sendFieldData(m, e, fields[nf], nf, shr); m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); receiveFieldData(fields,mismatch_fields); } } - int global_size = PCU_Max_Int((int)mismatch_fields.size()); + int global_size = m->getPCU()->Max((int)mismatch_fields.size()); if (global_size) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) for (std::set::iterator it=mismatch_fields.begin(); it!=mismatch_fields.end(); ++it) lion_oprint(1,"%s: \"%s\" DOF mismatch over remote/ghost copies\n", __func__, getName(*it)); } else { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"%s: no DOF mismatch\n", __func__); } } diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index 6fe0edb5b..97c01d5b5 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -54,7 +54,7 @@ pGeom pumi_geom_load(const char* filename, const char* model_type, void (*geom_l pGeom pumi_geom_load(gmi_model* gm, const char* model_type, const char* filename, void (*geom_load_fp)(const char*)) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (!strcmp(model_type,"null")) pumi::instance()->model = new gModel(gm); else if (!strcmp(model_type,"mesh")) @@ -78,7 +78,7 @@ pGeom pumi_geom_load(gmi_model* gm, const char* model_type, } if (!PCU_Comm_Self() && filename) - lion_oprint(1,"model %s loaded in %f seconds\n", filename, PCU_Time() - t0); + lion_oprint(1,"model %s loaded in %f seconds\n", filename, pcu::Time() - t0); return pumi::instance()->model; } diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc index e8dfdacdd..dff9f02e2 100644 --- a/pumi/pumi_ghost.cc +++ b/pumi/pumi_ghost.cc @@ -84,7 +84,7 @@ void Ghosting::send(pMeshEnt e, int to) /** assign a destination part id of all entities of dimension */ void Ghosting::send (int to) { - if (to==PCU_Comm_Self()) return; + if (to==m->getPCU()->Self()) return; pMeshEnt e; apf::MeshIterator* it = m->begin(ghost_dim); @@ -104,7 +104,7 @@ void Ghosting::print() m->getIntTag(e, parts_index_tag, &index); APF_ITERATE(Parts,*(parts_vec[ghost_dim][index]),pit) - std::cout<<"("<getPCU()->Self()<<") ghost e "<end(it); } @@ -135,9 +135,9 @@ int Ghosting::count() static pMeshEnt unpackGhost(Ghosting* plan, apf::DynamicArray& tags) // ********************************************************* { - int from = PCU_Comm_Sender(); + int from = plan->getMesh()->getPCU()->Sender(); int type; - PCU_COMM_UNPACK(type); + plan->getMesh()->getPCU()->Unpack(type); pMeshEnt sender; apf::ModelEntity* c; Parts residence; @@ -165,7 +165,7 @@ static void ghost_receiveEntities(Ghosting* plan, apf::DynamicArray& t // ********************************************************* { received.reserve(1024); - while (PCU_Comm_Receive()) + while (plan->getMesh()->getPCU()->Receive()) received.push_back(unpackGhost(plan,tags)); } @@ -173,7 +173,7 @@ static void ghost_receiveEntities(Ghosting* plan, apf::DynamicArray& t static void setupGhosts(pMesh m, EntityVector& received) // ********************************************************* { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,received,it) { pMeshEnt entity = *it; @@ -182,20 +182,20 @@ static void setupGhosts(pMesh m, EntityVector& received) apf::Copies temp; m->getGhosts(entity,temp); int to = temp.begin()->first; - PCU_COMM_PACK(to,temp.begin()->second); // sender - PCU_COMM_PACK(to,entity); + m->getPCU()->Pack(to,temp.begin()->second); // sender + m->getPCU()->Pack(to,entity); apf::Copies remotes; m->getRemotes(entity, remotes); APF_ITERATE(Copies,remotes,rit) { - PCU_COMM_PACK(rit->first, rit->second); // sender - PCU_COMM_PACK(rit->first,entity); + m->getPCU()->Pack(rit->first, rit->second); // sender + m->getPCU()->Pack(rit->first,entity); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); pMeshEnt entity; PCU_COMM_UNPACK(entity); pMeshEnt sender; @@ -265,7 +265,7 @@ static void ghost_collectEntities (pMesh m, Ghosting* plan, EntityVector entitie size_t msg_size; for (int dim = 0; dim <=ghost_dim; ++dim) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,entitiesToGhost[dim],it) { e = *it; @@ -289,18 +289,18 @@ static void ghost_collectEntities (pMesh m, Ghosting* plan, EntityVector entitie pids[pos]=*pit; ++pos; } - PCU_Comm_Write(rit->first, (void*)msg_send, msg_size); + m->getPCU()->Write(rit->first, (void*)msg_send, msg_size); free(msg_send); } } // for entitiesToGhost[dim] - PCU_Comm_Send(); + m->getPCU()->Send(); // receive phase void *msg_recv; int pid_from; int* pids; pMeshEnt r; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { r = *((pMeshEnt*)msg_recv); if ( !m->hasTag(r,tag)) @@ -341,7 +341,7 @@ void ghost_sendEntities(Ghosting* plan, int entDim, // ********************************************** { pMeshEnt ent; - int src_partid=PCU_Comm_Self(); + int src_partid=plan->getMesh()->getPCU()->Self(); pMesh m = plan->getMesh(); std::set res_parts, temp; @@ -394,7 +394,7 @@ void ghost_sendEntities(Ghosting* plan, int entDim, void pumi_ghost_create(pMesh m, Ghosting* plan) // ********************************************************* { - if (PCU_Comm_Peers()==1) { + if (m->getPCU()->Peers()==1) { delete plan; return; } @@ -411,7 +411,7 @@ void pumi_ghost_create(pMesh m, Ghosting* plan) } } - double t0=PCU_Time(); + double t0=pcu::Time(); EntityVector entities_to_ghost[4]; ghost_collectEntities(m, plan, entities_to_ghost); @@ -420,9 +420,9 @@ void pumi_ghost_create(pMesh m, Ghosting* plan) plan->getMesh()->getTags(tags); for (int dimension = 0; dimension <= plan->ghost_dim; ++dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); ghost_sendEntities(plan, dimension, entities_to_ghost[dimension], tags); - PCU_Comm_Send(); + m->getPCU()->Send(); EntityVector received; ghost_receiveEntities(plan,tags,received); setupGhosts(plan->getMesh(),received); @@ -439,8 +439,8 @@ void pumi_ghost_create(pMesh m, Ghosting* plan) for (std::vector::iterator fit=frozen_fields.begin(); fit!=frozen_fields.end(); ++fit) apf::freeze(*fit); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh ghosted in %f seconds\n", PCU_Time()-t0); + if (!m->getPCU()->Self()) + lion_oprint(1,"mesh ghosted in %f seconds\n", pcu::Time()-t0); } // ********************************************************* @@ -455,7 +455,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, pMeshEnt* s_ent; size_t msg_size; int dummy=1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int layer=2; layerfirst, (void*)msg_send, msg_size); + m->getPCU()->Write(brg_rit->first, (void*)msg_send, msg_size); free(msg_send); } // APF_ITERATE } // for (std::map >::iterator iter @@ -488,7 +488,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int j=0; jgetPCU()->Send(); // receive phase void *msg_recv; int pid_from; @@ -499,7 +499,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, std::vector processed_ent; std::vector adj_ent; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { processed_ent.clear(); @@ -583,7 +583,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, while (global_num_off_part) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int layer=0; layerfirst, (void*)msg_send, msg_size); + m->getPCU()->Write(brg_rit->first, (void*)msg_send, msg_size); free(msg_send); } // APF_ITERATE } // for off_it @@ -616,9 +616,9 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int j=0; jgetPCU()->Send(); - while (PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while (m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { processed_ent.clear(); @@ -709,7 +709,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, int include_copy) // ********************************************************* { - if (PCU_Comm_Peers()==1 || num_layer==0) return; + if (m->getPCU()->Peers()==1 || num_layer==0) return; int dummy=1, mesh_dim=m->getDimension(), self = pumi_rank();; @@ -722,7 +722,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, return; } - double t0 = PCU_Time(); + double t0 = pcu::Time(); pMeshTag tag = m->createIntTag("ghost_check_mark",1); Ghosting* plan = new Ghosting(m, ghost_dim); @@ -843,8 +843,8 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, // ******************************************** // STEP 3: perform ghosting // ******************************************** - if (!PCU_Comm_Self()) - lion_oprint(1,"ghosting plan computed in %f seconds\n", PCU_Time()-t0); + if (!m->getPCU()->Self()) + lion_oprint(1,"ghosting plan computed in %f seconds\n", pcu::Time()-t0); pumi_ghost_create(m, plan); } diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index b48ed473f..47737d87f 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -69,10 +69,10 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) ++num_own; m->end(it); - PCU_Exscan_Ints(&num_own,1); + m->getPCU()->Exscan(&num_own,1); int initial_id=num_own; - PCU_Comm_Begin(); + m->getPCU()->Begin(); it = m->begin(dim); while ((e = m->iterate(it))) { @@ -84,8 +84,8 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) m->getRemotes(e, remotes); APF_ITERATE(Copies, remotes, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_Comm_Pack(it->first, &initial_id, sizeof(int)); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, &initial_id, sizeof(int)); } if (m->isGhosted(e)) @@ -94,22 +94,22 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) m->getGhosts(e, ghosts); APF_ITERATE(Copies, ghosts, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_Comm_Pack(it->first, &initial_id, sizeof(int)); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, &initial_id, sizeof(int)); } } ++initial_id; } m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); int global_id; - while (PCU_Comm_Listen()) - while (!PCU_Comm_Unpacked()) + while (m->getPCU()->Listen()) + while (!m->getPCU()->Unpacked()) { pMeshEnt remote_ent; - PCU_COMM_UNPACK(remote_ent); - PCU_Comm_Unpack(&global_id, sizeof(int)); + m->getPCU()->Unpack(remote_ent); + m->getPCU()->Unpack(&global_id, sizeof(int)); m->setIntTag(remote_ent, tag, &global_id); } } @@ -261,17 +261,17 @@ pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* void send_entities(pMesh mesh, int dim) { - int local_id, self = PCU_Comm_Self(); + int local_id, self = mesh->getPCU()->Self(); pMeshEnt e; pMeshIter it = mesh->begin(dim); while ((e = mesh->iterate(it))) { local_id = getMdsIndex(mesh, e); - for (int pid=0; pidgetPCU()->Peers(); ++pid) { if (pid==self) continue; - PCU_Comm_Pack(pid, &local_id, sizeof(int)); - PCU_COMM_PACK(pid, e); + mesh->getPCU()->Pack(pid, &local_id, sizeof(int)); + mesh->getPCU()->Pack(pid, e); } } mesh->end(it); @@ -285,7 +285,7 @@ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename); else { - double t0 = PCU_Time(); + double t0 = pcu::Time(); MPI_Comm prevComm = PCU_Get_Comm(); int num_target_part = PCU_Comm_Peers(); split_comm(num_target_part); @@ -293,7 +293,7 @@ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename); merge_comm(prevComm); if (!PCU_Comm_Self()) - lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, PCU_Time() - t0); + lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); } if (pumi_size()>1 && stitch_link) @@ -357,7 +357,7 @@ int pumi_mesh_getNumEnt(pMesh m, int dim) int pumi_mesh_getNumOwnEnt(pMesh m, int dim) { PCU_ALWAYS_ASSERT(pumi::instance()->num_own_ent); - if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !PCU_Comm_Self()) + if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !m->getPCU()->Self()) { std::cout<<"[PUMI ERROR] "<<__func__<<": mesh count is not set. Please call pumi_mesh_setCount\n"; return -1; @@ -368,7 +368,7 @@ int pumi_mesh_getNumOwnEnt(pMesh m, int dim) int pumi_mesh_getNumGlobalEnt(pMesh m, int dim) { PCU_ALWAYS_ASSERT(pumi::instance()->num_global_ent); - if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !PCU_Comm_Self()) + if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !m->getPCU()->Self()) { std::cout<<"[PUMI ERROR] "<<__func__<<": mesh count is not set. Please call pumi_mesh_setCount\n"; return -1; @@ -407,12 +407,12 @@ void print_copies(pMesh m, pMeshEnt e) void pumi_mesh_print (pMesh m, bool print_ent) { - if (!PCU_Comm_Self()) std::cout<<"\n=== mesh size and tag info === \n"; + if (!m->getPCU()->Self()) std::cout<<"\n=== mesh size and tag info === \n"; - int* local_entity_count = new int[4*PCU_Comm_Peers()]; - int* own_entity_count = new int[4*PCU_Comm_Peers()]; + int* local_entity_count = new int[4*m->getPCU()->Peers()]; + int* own_entity_count = new int[4*m->getPCU()->Peers()]; - for (int i=0; i<4*PCU_Comm_Peers();++i) + for (int i=0; i<4*m->getPCU()->Peers();++i) local_entity_count[i]=own_entity_count[i]=0; pMeshEnt e; @@ -430,23 +430,23 @@ void pumi_mesh_print (pMesh m, bool print_ent) m->end(it); } - int* global_local_entity_count = new int[4*PCU_Comm_Peers()]; - int* global_own_entity_count = new int[4*PCU_Comm_Peers()]; + int* global_local_entity_count = new int[4*m->getPCU()->Peers()]; + int* global_own_entity_count = new int[4*m->getPCU()->Peers()]; - MPI_Allreduce(local_entity_count, global_local_entity_count, 4*PCU_Comm_Peers(), + MPI_Allreduce(local_entity_count, global_local_entity_count, 4*m->getPCU()->Peers(), MPI_INT, MPI_SUM, PCU_Get_Comm()); - MPI_Allreduce(own_entity_count, global_own_entity_count, 4*PCU_Comm_Peers(), + MPI_Allreduce(own_entity_count, global_own_entity_count, 4*m->getPCU()->Peers(), MPI_INT, MPI_SUM, PCU_Get_Comm()); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) { int* global_entity_count = new int[4]; global_entity_count[0]=global_entity_count[1]=global_entity_count[2]=global_entity_count[3]=0; for (int d=0; d<4;++d) { - for (int p=0; pgetPCU()->Peers();++p) global_entity_count[d] += global_own_entity_count[p*4+d]; } @@ -455,12 +455,12 @@ void pumi_mesh_print (pMesh m, bool print_ent) delete [] global_entity_count; - for (int p=0; pgetPCU()->Peers(); ++p) std::cout<<"(p"<getPCU()->Peers(); ++p) if (global_own_entity_count[p*4]) std::cout<<"(p"<getPCU()->Self()) { std::cout<<"mesh shape: \""<< m->getShape()->getName()<<"\"\n"; @@ -508,7 +508,7 @@ void pumi_mesh_print (pMesh m, bool print_ent) if (!m->findTag("global_id")) { pumi_mesh_createGlobalID(m); - if (!PCU_Comm_Self()) std::cout<<__func__<<": global id generated\n"; + if (!m->getPCU()->Self()) std::cout<<__func__<<": global id generated\n"; } int global_id; @@ -518,10 +518,10 @@ void pumi_mesh_print (pMesh m, bool print_ent) apf::Vector3 xyz; m->getPoint(e, 0, xyz); if (m->isGhost(e)) - std::cout<<"("<getPCU()->Self()<<") GHOST vtx "<getPCU()->Self()<<") vtx "<getDownward(e,d-1,down); if (m->isGhost(e)) - std::cout<<"("<getPCU()->Self()<<") GHOST e "<getPCU()->Self()<<") e "<getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Self()) { ++num_own_ent; if (own_copy!=e) @@ -753,7 +753,7 @@ void Distribution::print() ++i; if (parts_vec[i].size()==0) continue; APF_ITERATE(Parts,parts_vec[i],pit) - std::cout<<"("<getPCU()->Self()<<") distribute element "<end(it); @@ -780,7 +780,7 @@ static void distr_getAffected (pMesh m, Distribution* plan, EntityVector affecte for (int dimension=maxDimension-1; dimension >= 0; --dimension) { int upDimension = dimension + 1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[upDimension],it) { pMeshEnt up = *it; @@ -796,18 +796,18 @@ static void distr_getAffected (pMesh m, Distribution* plan, EntityVector affecte Copies remotes; m->getRemotes(adjacent[i],remotes); APF_ITERATE(Copies,remotes,rit) - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); if (m->hasMatching()) { apf::Matches matches; m->getMatches(adjacent[i],matches); for (size_t j=0; j < matches.getSize(); ++j) - PCU_COMM_PACK(matches[j].peer,matches[j].entity); + m->getPCU()->Pack(matches[j].peer,matches[j].entity); } }//downward adjacent loop }//upward affected loop - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { pMeshEnt entity; PCU_COMM_UNPACK(entity); @@ -854,7 +854,7 @@ static void distr_updateResidences(pMesh m, for (int dimension = maxDimension-1; dimension >= 0; --dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[dimension],it) { pMeshEnt entity = *it; @@ -873,12 +873,12 @@ static void distr_updateResidences(pMesh m, m->getRemotes(entity,remotes); APF_ITERATE(Copies,remotes,rit) { - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); apf::packParts(rit->first,newResidence); } } - PCU_Comm_Send(); - while(PCU_Comm_Receive()) + m->getPCU()->Send(); + while(m->getPCU()->Receive()) { pMeshEnt entity; PCU_COMM_UNPACK(entity); @@ -913,11 +913,11 @@ void distribute(pMesh m, Distribution* plan) void pumi_mesh_distribute(pMesh m, Distribution* plan) // ********************************************************* { - if (PCU_Comm_Peers()==1) return; + if (m->getPCU()->Peers()==1) return; if (pumi::instance()->ghosted_tag) { - if (!PCU_Comm_Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" not supported with ghosted mesh\n"; + if (!m->getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" not supported with ghosted mesh\n"; return; } distribute(m, plan); diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index e8997231b..f1a5a04d7 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -99,7 +99,7 @@ pNumbering pumi_numbering_createProcGrp ( pMesh m, const char* name, int num_proc_grp, int dim, pOwnership o) { - assert(PCU_Comm_Peers()%num_proc_grp==0); + assert(m->getPCU()->Peers()%num_proc_grp==0); pNumbering n = m->findNumbering(name); if (n) @@ -109,8 +109,8 @@ pNumbering pumi_numbering_createProcGrp ( return n; } - int self = PCU_Comm_Self(); - int pgrp_size = PCU_Comm_Peers()/num_proc_grp; + int self = m->getPCU()->Self(); + int pgrp_size = m->getPCU()->Peers()/num_proc_grp; int local_pgrpid = self/pgrp_size; // divide int pgrp_rank = self % pgrp_size; @@ -135,7 +135,7 @@ pNumbering pumi_numbering_createProcGrp ( } int* in = new int; - int* out_arr = new int[PCU_Comm_Peers()]; // out[i] has local_numOwnedPartBdryEnt of process i on all processes + int* out_arr = new int[m->getPCU()->Peers()]; // out[i] has local_numOwnedPartBdryEnt of process i on all processes *in = owned_node_cnt; MPI_Allgather(in, 1, MPI_INT, out_arr, 1, MPI_INT, PCU_Get_Comm()); @@ -194,10 +194,10 @@ void pumi_numbering_print(pNumbering n, int pid) pMeshEnt e; int nnodes; pShape s = n->getShape(); - if (pid==-1) pid = PCU_Comm_Peers(); + if (pid==-1) pid = m->getPCU()->Peers(); for (int rank=0; rankgetPCU()->Self()) { for(int dd = 0; dd < m->getDimension(); ++dd) { @@ -208,7 +208,7 @@ void pumi_numbering_print(pNumbering n, int pid) { nnodes = n->countNodesOn(e); for (int node=0; node < nnodes; ++node) - std::cout<<"("<getPCU()->Self()<<") ent "<isGhost(e)?1:0)<<")\n"; } // while m->end(it); From b1e41ff04cd0d5753f54d6eb4beebaa0cf3743cf Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 25 Jan 2024 11:54:58 -0500 Subject: [PATCH 038/141] more cleaning --- pumi/pumi_sys.cc | 2 +- ree/reeEstimateError.cc | 8 +- ree/reeResidualFunctionals.cc | 924 ---------------------------------- ree/reeSizeField.cc | 6 +- 4 files changed, 8 insertions(+), 932 deletions(-) diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc index b41be4ff3..cc46b17ca 100644 --- a/pumi/pumi_sys.cc +++ b/pumi/pumi_sys.cc @@ -64,7 +64,7 @@ double pumi_getTime() double pumi_getMem() { - return PCU_GetMem(); + return pcu::GetMem(); } void pumi_printTimeMem(const char* msg, double time, double memory) diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc index c3a4ab060..183713d0e 100644 --- a/ree/reeEstimateError.cc +++ b/ree/reeEstimateError.cc @@ -431,13 +431,13 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux) apf::Field* estimateError(apf::Field* f) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); apf::Field* g = ree::equilibrateResiduals(f); lion_eprint(1,"1/4: residuals equilibrated \n"); apf::Field* theta = ree::computeFluxCorrection(f, g); lion_eprint(1,"2/4: flux corrections computed \n"); apf::destroyField(g); - PCU_Barrier(); + f->getMesh()->getPCU()->Barrier(); apf::Field* correctedFlux = ree::computeCorrectedFlux(f, theta); lion_eprint(1,"3/4: corrected flux field computed\n"); @@ -447,8 +447,8 @@ apf::Field* estimateError(apf::Field* f) lion_eprint(1,"4/4: error computed \n"); apf::destroyField(correctedFlux); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!f->getMesh()->getPCU()->Self()) lion_eprint(1,"REE: Error estimated in %f seconds\n",t1-t0); return error_field; diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc index f26adea88..e69de29bb 100644 --- a/ree/reeResidualFunctionals.cc +++ b/ree/reeResidualFunctionals.cc @@ -1,924 +0,0 @@ -/* - * Copyright (C) 2011 Scientific Computation Research Center - * - * This work is open source software, licensed under the terms of the - * BSD license as described in the LICENSE file in the top-level directory. - */ -#include -#include -#include "apfElement.h" -#include "ree.h" - -namespace ree { - -enum {VISITED}; - -/* overall information useful during equilibration */ -struct Equilibration { - apf::Mesh* mesh; - /* mesh dimension, so far handling 3 only */ - int dim; - /* polynomial order of Nedelec space */ - int order; - /* input scalar field containing Nedelec dofs for solution electric field */ - apf::Field* ef; - /* tags each edge once it has been visited during the equilibration process */ - apf::MeshTag* tag; - /* output field containing correction values. - * currently 3 scalar values are stored on each face - * in order corresponding to downward edges of the face */ - apf::Field* g; -}; - -static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g) -{ - eq->mesh = apf::getMesh(f); - eq->tag = eq->mesh->createIntTag("isVisited", 1); - eq->dim = eq->mesh->getDimension(); - eq->ef = f; - eq->order = f->getShape()->getOrder(); - eq->g = g; - - double zeros[3] = {0., 0., 0.}; - apf::MeshEntity* face; - apf::MeshIterator* it = eq->mesh->begin(2); - while ((face = eq->mesh->iterate(it))) { - apf::setComponents(eq->g, face, 0, zeros); - } - eq->mesh->end(it); -} - -struct QRDecomp { - mth::Matrix Q; - mth::Matrix R; -}; - -typedef std::vector EntityVector; - -struct EdgePatch { - apf::Mesh* mesh; - Equilibration* equilibration; - /* the edge entity around which the patch - is centered. a patch collects entities - (faces & tets) around this edge entity */ - apf::MeshEntity* entity; - bool isOnBdry; - EntityVector tets; - EntityVector faces; - mth::Matrix A; - mth::Matrix At; - mth::Matrix T; // T = A*At + 1 - mth::Vector b; - mth::Vector x; - QRDecomp qr; -}; - -static void setupEdgePatch(EdgePatch* ep, Equilibration* eq) -{ - ep->mesh = eq->mesh; - ep->equilibration = eq; - ep->entity = 0; -} - -static void startEdgePatch(EdgePatch* ep, apf::MeshEntity* e) -{ - ep->tets.clear(); - ep->faces.clear(); - ep->entity = e; - ep->isOnBdry = isOnDomainBoundary(ep->mesh, ep->entity); -} - -static void addEntityToPatch(EdgePatch* ep, apf::MeshEntity* e) -{ - if(ep->mesh->getType(e) == apf::Mesh::TRIANGLE) - ep->faces.push_back(e); - if(ep->mesh->getType(e) == apf::Mesh::TET) - ep->tets.push_back(e); -} - -static void addEntitiesToPatch( - EdgePatch* ep, apf::DynamicArray& es) -{ - for (std::size_t i=0; i < es.getSize(); ++i) - addEntityToPatch(ep, es[i]); -} - -static bool getInitialEdgePatch(EdgePatch* ep, apf::CavityOp* o) -{ - if ( ! o->requestLocality(&ep->entity,1)) - return false; - apf::DynamicArray adjacent; - ep->mesh->getAdjacent(ep->entity, 3, adjacent); - addEntitiesToPatch(ep, adjacent); - - ep->mesh->getAdjacent(ep->entity, 2, adjacent); - addEntitiesToPatch(ep, adjacent); - - return true; -} - -static bool buildEdgePatch(EdgePatch* ep, apf::CavityOp* o) -{ - if (!getInitialEdgePatch(ep, o)) return false; - return true; -} - -/* - * Reference: Leszek Demkowicz, Computing with hp-adaptive - * finite elements, vol2, equation (11.118, 11.119, 11.122). - */ -static void assembleEdgePatchLHS(EdgePatch* ep) -{ - int ne = ep->tets.size(); - int nf = ep->faces.size(); - if( isOnDomainBoundary(ep->mesh, ep->entity) ) { - ep->T.resize(ne+nf, ne+nf); - ep->T.zero(); - for (int i = 0; i < nf; i++) - ep->T(i,i) = 2.; - for (int i = 0; i < ne-1; i++) { - ep->T(i+nf,i) = 1.; ep->T(i+nf,i+1) = -1.; - ep->T(i,i+nf) = 1.; ep->T(i+1,i+nf) = -1.; - } - ep->T(ne+nf-1, ne-1) = 1.; ep->T(ne+nf-1, ne) = 1.; - ep->T(ne-1, ne+nf-1) = 1.; ep->T(ne, ne+nf-1) = 1.; - } - else if( ! isOnDomainBoundary(ep->mesh, ep->entity) ) { - ep->A.resize(ne, nf); - ep->A.zero(); - for (int i = 0; i < ne-1; i++) { - ep->A(i,i) = 1.; ep->A(i,i+1) = -1.; - } - ep->A(ne-1,0) = -1.; ep->A(ne-1,ne-1) = 1.; - // A is singular so do (A*At) + 1.0 - // to pick a particular solution - ep->At.resize(nf,ne); - mth::transpose(ep->A, ep->At); - mth::multiply(ep->A, ep->At, ep->T); - - for (int i = 0; i < ne; i++) - for (int j = 0; j < ne; j++) - ep->T(i,j) += 1.; - } - mth::decomposeQR(ep->T, ep->qr.Q, ep->qr.R); -} - -/* - * Performs Curl Curl integration using curl vector Nedelec shapes - */ -void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e, - apf::Field* f, mth::Matrix& elmat) -{ - apf::FieldShape* fs = f->getShape(); - int type = mesh->getType(e); - PCU_ALWAYS_ASSERT(type == apf::Mesh::TET); - int nd = apf::countElementNodes(fs, type); - int dim = apf::getDimension(mesh, e); - int dimc = (dim == 3) ? 3 : 1; - double w; - - apf::NewArray curlshape(nd); - mth::Matrix phys_curlshape(nd, dimc); - elmat.resize(nd,nd); - - apf::MeshElement* me = apf::createMeshElement(mesh, e); - apf::Element* el = apf::createElement(f, me); - int int_order = 2 * fs->getOrder() - 2; - int np = apf::countIntPoints(me, int_order); - - elmat.zero(); - apf::Vector3 p; - for (int i = 0; i < np; i++) { - apf::getIntPoint(me, int_order, i, p); - double weight = apf::getIntWeight(me, int_order, i); - apf::Matrix3x3 J; - apf::getJacobian(me, p, J); - double jdet = apf::getJacobianDeterminant(J, dim); - w = weight / jdet; - - if (dim == 3) { - el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape); - phys_curlshape.zero(); - for (int j = 0; j < nd; j++) - for (int k = 0; k < dim; k++) - for (int l = 0; l < dim; l++) - phys_curlshape(j,k) += curlshape[j][l] * J[l][k]; - } - mth::Matrix phys_curlshapeT; - mth::transpose(phys_curlshape, phys_curlshapeT); - - mth::Matrix M (nd, nd); - M.zero(); - mth::multiply(phys_curlshape, phys_curlshapeT, M); - M *= w; - elmat += M; - } - apf::destroyElement(el); - apf::destroyMeshElement(me); -} - -/* - * Performs Vector Vector Mass integration using vector Nedelec shapes - */ -void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e, - apf::Field* f, mth::Matrix& elmat) -{ - apf::FieldShape* fs = f->getShape(); - int type = mesh->getType(e); - PCU_ALWAYS_ASSERT(type == apf::Mesh::TET || type == apf::Mesh::TRIANGLE); - int nd = apf::countElementNodes(fs, type); - int dim = apf::getDimension(mesh, e); - int sdim = mesh->getDimension(); - double w; - - apf::NewArray vectorshapes(nd); - elmat.resize(nd,nd); - - apf::MeshElement* me = apf::createMeshElement(mesh, e); - apf::Element* el = apf::createElement(f, me); - int int_order = 2 * fs->getOrder(); - int np = apf::countIntPoints(me, int_order); - - elmat.zero(); - apf::Vector3 p; - for (int i = 0; i < np; i++) { - apf::getIntPoint(me, int_order, i, p); - double weight = apf::getIntWeight(me, int_order, i); - apf::Matrix3x3 J; - apf::getJacobian(me, p, J); - double jdet = apf::getJacobianDeterminant(J, dim); - w = weight * jdet; - - apf::getVectorShapeValues(el, p, vectorshapes); - mth::Matrix vectorShapes (nd, sdim); - for (int j = 0; j < nd; j++) - for (int k = 0; k < sdim; k++) - vectorShapes(j,k) = vectorshapes[j][k]; - - mth::Matrix vectorShapesT (sdim, nd); - mth::transpose(vectorShapes, vectorShapesT); - - mth::Matrix M (nd,nd); - M.zero(); - mth::multiply(vectorShapes, vectorShapesT, M); - M *= w; - elmat += M; - } - - apf::destroyElement(el); - apf::destroyMeshElement(me); -} - -void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e, - apf::Field* f, mth::Matrix& elmat) -{ - mth::Matrix curl_elmat, mass_elmat; - assembleCurlCurlElementMatrix(mesh, e, f, curl_elmat); - assembleVectorMassElementMatrix(mesh, e, f, mass_elmat); - - elmat.resize(curl_elmat.rows(), curl_elmat.cols()); - elmat.zero(); - elmat += curl_elmat; - elmat += mass_elmat; -} - -/* - * computes local bilinear form integral restricted to - * an edge of a tet element - */ -static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet) -{ - PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET); - // findIn edge in downward edges of tet - apf::Downward e; - int ne = ep->mesh->getDownward(tet, 1, e); - int ei = apf::findIn(e, ne, ep->entity); - // get Element Dofs - apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet); - apf::Element* el = apf::createElement(ep->equilibration->ef, me); - int type = ep->mesh->getType(tet); - int nd = apf::countElementNodes(el->getFieldShape(), type); - apf::NewArray d (nd); - el->getElementNodeData(d); - mth::Vector dofs (nd); - for (int i = 0; i < nd; i++) - dofs(i) = d[i]; - // assemble curl curl element matrix - mth::Matrix curl_elmat; - assembleCurlCurlElementMatrix(ep->mesh, tet, - ep->equilibration->ef, curl_elmat); - // assemble vector mass element matrix - mth::Matrix mass_elmat; - assembleVectorMassElementMatrix(ep->mesh, tet, - ep->equilibration->ef, mass_elmat); - // add element matrices - mth::Matrix elmat(nd, nd); - elmat.zero(); - elmat += curl_elmat; - elmat += mass_elmat; - // multiply element matrix with element dofs - mth::Vector blf_integrals (nd); - mth::multiply(elmat, dofs, blf_integrals); - - apf::destroyElement(el); - apf::destroyMeshElement(me); - - // pick edge index from the resulting vector - // negation of negative ND dofs - int which, rotate; bool flip; - apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate); - if (flip) - blf_integrals(ei) = -1*blf_integrals(ei); - return blf_integrals(ei); -} - -/* - * Capability can be added in the future to allow the user - * to define a custom function */ -void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e, - const apf::Vector3& x, mth::Vector& f) -{ - double freq = 1.; - double kappa = freq * M_PI; - int dim = apf::getDimension(mesh, e); - if (dim == 3) { - f(0) = (1. + kappa * kappa) * sin(kappa * x[1]); - f(1) = (1. + kappa * kappa) * sin(kappa * x[2]); - f(2) = (1. + kappa * kappa) * sin(kappa * x[0]); - /*f(0) = 0.; - f(1) = 0.; - f(2) = 0.;*/ - } - else { - f(0) = (1. + kappa * kappa) * sin(kappa * x[1]); - f(1) = (1. + kappa * kappa) * sin(kappa * x[0]); - f(2) = 0.0; - } -} -void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e, - apf::Field* f, mth::Vector& elvect) -{ - apf::FieldShape* fs = f->getShape(); - int type = mesh->getType(e); - PCU_ALWAYS_ASSERT(type == apf::Mesh::TET); - int nd = apf::countElementNodes(fs, type); - int dim = apf::getDimension(mesh, e); - double w; - - apf::NewArray vectorshapes(nd); - elvect.resize(nd); - mth::Vector val (dim); - val.zero(); - apf::Vector3 p; - - apf::MeshElement* me = apf::createMeshElement(mesh, e); - apf::Element* el = apf::createElement(f, me); - int int_order = 2 * fs->getOrder(); - int np = apf::countIntPoints(me, int_order); - - elvect.zero(); - for (int i = 0; i < np; i++) { - apf::getIntPoint(me, int_order, i, p); - double weight = apf::getIntWeight(me, int_order, i); - apf::Matrix3x3 J; - apf::getJacobian(me, p, J); - double jdet = apf::getJacobianDeterminant(J, dim); - w = weight * jdet; - - apf::getVectorShapeValues(el, p, vectorshapes); - apf::Vector3 global; - apf::mapLocalToGlobal(me, p, global); - pumiUserFunction(mesh, e, global, val); - val *= w; - - mth::Matrix vectorShapes (nd, dim); - for (int j = 0; j < nd; j++) - for (int k = 0; k < dim; k++) - vectorShapes(j,k) = vectorshapes[j][k]; - mth::Vector V (nd); - V.zero(); - mth::multiply(vectorShapes, val, V); - elvect += V; - } - - apf::destroyElement(el); - apf::destroyMeshElement(me); -} - -/* - * computes local linear form integral restricted to - * an edge of a tet element - */ -static double getLocalEdgeLF(EdgePatch* ep, apf::MeshEntity* tet) -{ - PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET); - // findIn edge in downward edges of tet - apf::Downward e; - int ne = ep->mesh->getDownward(tet, 1, e); - int ei = apf::findIn(e, ne, ep->entity); - // assemble Domain LF Vector - mth::Vector elvect; - assembleDomainLFElementVector(ep->mesh, tet, - ep->equilibration->ef, elvect); - int which, rotate; bool flip; - apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate); - if (flip) - elvect(ei) = -1*elvect(ei); - return elvect(ei); -} - -/* - * Given a tet and one of its faces, the vertex of the tet - * opposite to the given face is returned. - */ -static apf::MeshEntity* getTetOppVert( - apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f) -{ - apf::Downward fvs; - int fnv = m->getDownward(f, 0, fvs); - apf::Downward tvs; - int tnv = m->getDownward(t, 0, tvs); - PCU_ALWAYS_ASSERT(tnv == 4 && fnv == 3); - for (int i = 0; i < tnv; i++) { - if (apf::findIn(fvs, fnv, tvs[i]) == -1) - return tvs[i]; - } - return 0; -} - -/* - * Given a face and one of its edges, the vertex of the face - * opposite to the given edge is returned. - */ -static apf::MeshEntity* getFaceOppVert( - apf::Mesh* m, apf::MeshEntity* f, apf::MeshEntity* e) -{ - apf::Downward evs; - int env = m->getDownward(e, 0, evs); - apf::Downward fvs; - int fnv = m->getDownward(f, 0, fvs); - PCU_ALWAYS_ASSERT(env == 2 && fnv == 3); - for (int i = 0; i < fnv; i++) { - if (apf::findIn(evs, env, fvs[i]) == -1) - return fvs[i]; - } - return 0; -} - -static apf::Vector3 computeFaceNormal( - apf::Mesh* m, apf::MeshEntity* f, apf::Vector3 const& p) -{ - // Compute face normal using face Jacobian, - // so it can also be used for curved mesh elements - apf::MeshElement* me = apf::createMeshElement(m, f); - apf::Matrix3x3 J; - apf::getJacobian(me, p, J); - apf::destroyMeshElement(me); - - apf::Vector3 g1 = J[0]; - apf::Vector3 g2 = J[1]; - apf::Vector3 n = apf::cross( g1, g2 ); - return n.normalize(); -} - -apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m, - apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p) -{ - apf::Vector3 n = computeFaceNormal(m, f, p); - - // orient the normal outwards from the tet - apf::MeshEntity* oppVert = getTetOppVert(m, t, f); - apf::Vector3 vxi = apf::Vector3(0.,0.,0.); - - apf::Vector3 txi; - m->getPoint(oppVert, 0, txi); - - apf::MeshElement* fme = apf::createMeshElement(m, f); - apf::Vector3 pxi; - apf::mapLocalToGlobal(fme, p, pxi); - apf::destroyMeshElement(fme); - - apf::Vector3 pxiTotxi = txi - pxi; - if (pxiTotxi*n > 0) { - n = n*-1.; - } - return n; -} - -// TODO this needs generalisation for cases with internal model boundaries -// (i.e., interfaces) -bool isOnDomainBoundary(apf::Mesh* m, apf::MeshEntity* e) -{ - return m->getModelType(m->toModel(e)) < m->getDimension(); -} - -/* - * computes local flux integral restricted to - * an edge of a tet element - */ -static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet) -{ - PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET); - - double fluxIntegral = 0.0; - // 1. get faces of the tet in the patch - apf::Downward f; - int nf = ep->mesh->getDownward(tet, 2, f); - std::vector patchFaces; - for (int i = 0; i < nf; i++) { - if(std::find(ep->faces.begin(), ep->faces.end(), f[i]) != ep->faces.end()) - patchFaces.push_back(f[i]); - } - PCU_ALWAYS_ASSERT(patchFaces.size() == 2); - - // 2. loop over the patch faces - for (unsigned int i = 0; i < patchFaces.size(); i++) { - double fluxFaceIntegral = 0.0; - // 3. get upward tets of the current face - apf::Up up; - apf::MeshEntity* currentFace = patchFaces[i]; - ep->mesh->getUp(currentFace, up); - if (isOnDomainBoundary(ep->mesh, currentFace)) - PCU_ALWAYS_ASSERT( up.n == 1); - else - PCU_ALWAYS_ASSERT( up.n == 2); - - apf::MeshEntity* firstTet = up.e[0]; - apf::MeshEntity* secondTet = nullptr; - if (up.n == 2) - secondTet = up.e[1]; - - // 4. findIn edge in downward edges of current face - apf::Downward e; - int ne = ep->mesh->getDownward(currentFace, 1, e); - PCU_ALWAYS_ASSERT(ne == 3); - int ei = apf::findIn(e, ne, ep->entity); - - // 5. count integration points for flux face integral - apf::FieldShape* fs = ep->equilibration->ef->getShape(); - int int_order = 2 * fs->getOrder(); - apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace); - apf::Element* fel = apf::createElement(ep->equilibration->ef, fme); - int np = apf::countIntPoints(fme, int_order); - - // loop over integration points - apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl, - fn1, fn2, tk, vshape; - for (int n = 0; n < np; n++) { - apf::getIntPoint(fme, int_order, n, p); - double weight = apf::getIntWeight(fme, int_order, n); - apf::Matrix3x3 fJ; - apf::getJacobian(fme, p, fJ); - double jdet = apf::getJacobianDeterminant( - fJ, apf::getDimension(ep->mesh, currentFace)); - - // compute face outward normals wrt tets - if (tet == firstTet) { - fn1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p); - fn2 = apf::Vector3(0.,0.,0.); - } - else { - fn1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p); - fn2 = apf::Vector3(0.,0.,0.); - } - if (up.n == 2) { - if (tet == firstTet) - fn2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p); - else - fn2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p); - } - - curl.zero(); - // compute curl1 - tet1xi = apf::boundaryToElementXi(ep->mesh, currentFace, firstTet, p); - apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet); - apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1); - apf::getCurl(el1, tet1xi, curl1); - apf::Vector3 temp1 = apf::cross(fn1, curl1); - curl += temp1; - apf::destroyElement(el1); - apf::destroyMeshElement(me1); - - // compute curl2 - if (up.n == 2) { - tet2xi = apf::boundaryToElementXi(ep->mesh, currentFace, secondTet, p); - apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet); - apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2); - apf::getCurl(el2, tet2xi, curl2); - apf::Vector3 temp2 = apf::cross(fn2, curl2); - curl += (temp2 * -1.); - curl = curl * 1./2.; - apf::destroyElement(el2); - apf::destroyMeshElement(me2); - } - - // compute tk (inter-element averaged flux) - tk = curl; - - // compute vector shape - int type = apf::Mesh::TRIANGLE; - int nd = apf::countElementNodes(fs, type); - apf::NewArray vectorshapes (nd); - apf::getVectorShapeValues(fel, p, vectorshapes); - vshape = vectorshapes[ei]; - - // compute integral - fluxFaceIntegral += (tk * vshape) * weight * jdet; - } - apf::destroyElement(fel); - apf::destroyMeshElement(fme); - - fluxIntegral += fluxFaceIntegral; - } - return fluxIntegral; -} - -static void assembleEdgePatchRHS(EdgePatch* p) -{ - if (p->isOnBdry) { - p->b.resize(p->tets.size() + p->faces.size()); - p->b.zero(); - } - else { - p->b.resize(p->tets.size()); - p->b.zero(); - } - int ne = p->tets.size(); - int nf = p->faces.size(); - for (int i = 0; i < ne; i++) { - apf::MeshEntity* tet = p->tets[i]; - double blfIntegral = getLocalEdgeBLF(p, tet); - double lfIntegral = getLocalEdgeLF(p, tet); - double fluxIntegral = getLocalFluxIntegral(p, tet); - if(p->isOnBdry) - p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral; - else - p->b(i) = blfIntegral - lfIntegral - fluxIntegral; - } -} - -static apf::MeshEntity* getTetOppFaceSharingEdge( - apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f, apf::MeshEntity* e) -{ - apf::MeshEntity* fs[4]; - m->getDownward(t, 2, fs); - for (int i = 0; i < 4; i++) { - if (fs[i] == f) continue; - apf::MeshEntity* es[3]; - m->getDownward(fs[i], 1, es); - if (apf::findIn(es, 3, e) > -1) - return fs[i]; - } - return 0; -} - -/* - * Orders tets and faces in an edge cavity in a - * clockwise (or ccw) direction around the edge. - */ -static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge, - EntityVector& tets, EntityVector& faces) -{ - tets.clear(); - faces.clear(); - if( ! isOnDomainBoundary(mesh, edge) ) { - apf::MeshEntity* currentFace = mesh->getUpward(edge, 0); - apf::Up up; - mesh->getUp(currentFace, up); - PCU_ALWAYS_ASSERT(up.n == 2); - apf::MeshEntity* firstTet = up.e[0]; - apf::MeshEntity* nextTet = up.e[1]; - tets.push_back(firstTet); - apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet, - currentFace, edge); - faces.push_back(firstFace); - - while (nextTet != firstTet) { - tets.push_back(nextTet); - faces.push_back(currentFace); - currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge); - PCU_ALWAYS_ASSERT(currentFace); - apf::Up up; - mesh->getUp(currentFace, up); - PCU_ALWAYS_ASSERT(up.n == 2); - if (nextTet != up.e[0]) - nextTet = up.e[0]; - else - nextTet = up.e[1]; - } - } - else { - apf::Up up; - mesh->getUp(edge, up); - apf::MeshEntity* firstFace = nullptr; - for (int i = 0; i < up.n; i++) { - if ( isOnDomainBoundary(mesh, up.e[i]) ) { - firstFace = up.e[i]; break; - } - } - PCU_ALWAYS_ASSERT(firstFace); - faces.push_back(firstFace); - mesh->getUp(firstFace, up); - PCU_ALWAYS_ASSERT(up.n == 1); - apf::MeshEntity* firstTet = up.e[0]; - tets.push_back(firstTet); - - apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet, - firstFace, edge); - apf::MeshEntity* nextTet = firstTet; - mesh->getUp(nextFace, up); - while( up.n == 2) { - faces.push_back(nextFace); - if (nextTet != up.e[0]) - nextTet = up.e[0]; - else - nextTet = up.e[1]; - tets.push_back(nextTet); - - nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge); - mesh->getUp(nextFace, up); - } - faces.push_back(nextFace); - } -} - -void getClockwiseTetsandFaces(EdgePatch* p) -{ - if (p->isOnBdry) { - if (p->tets.size() > 1) { - apf::MeshEntity* secondFace = p->faces[1]; - apf::MeshEntity* firstTet = p->tets[0]; - apf::MeshEntity* secondTet = p->tets[1]; - - apf::Downward firstTetFaces, secondTetFaces; - int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces); - int n_secondTetFaces = p->mesh->getDownward(secondTet, 2, secondTetFaces); - int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, secondFace); - int fi2 = apf::findIn(secondTetFaces, n_secondTetFaces, secondFace); - PCU_ALWAYS_ASSERT(fi1 != -1 && fi2 != -1); - - // first tet opp vertex crd - apf::MeshEntity* firstTetOppVert = getTetOppVert( - p->mesh, firstTet, secondFace); - apf::Vector3 firstTetOppVertCrd; - p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd); - - // second tet opp vertex crd - apf::MeshEntity* secondTetOppVert = getTetOppVert( - p->mesh, secondTet, secondFace); - apf::Vector3 secondTetOppVertCrd; - p->mesh->getPoint(secondTetOppVert, 0, secondTetOppVertCrd); - - // normal to the face - apf::Downward edge_vertices; - p->mesh->getDownward(p->entity, 0, edge_vertices); - apf::Vector3 p0, p1, p2; - p->mesh->getPoint(edge_vertices[0], 0, p0); - p->mesh->getPoint(edge_vertices[1], 0, p1); - apf::MeshEntity* faceOppVert = getFaceOppVert( - p->mesh, secondFace, p->entity); - p->mesh->getPoint(faceOppVert, 0, p2); - - apf::Vector3 normal = apf::cross(p1-p0, p2-p0); - - // direction vectors from p0 to opp tet verts - apf::Vector3 vFirst = firstTetOppVertCrd - p0; - apf::Vector3 vLast = secondTetOppVertCrd - p0; - - if ((vFirst * normal > 0) && (vLast * normal < 0)) { - // reverse list of tets and faces - std::reverse(p->tets.begin(), p->tets.end()); - std::reverse(p->faces.begin(), p->faces.begin()); - } - } - } - else { - apf::MeshEntity* firstFace = p->faces[0]; - apf::MeshEntity* firstTet = p->tets[0]; - apf::MeshEntity* lastTet = p->tets[p->tets.size()-1]; - - apf::Downward firstTetFaces, lastTetFaces; - int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces); - int n_lastTetFaces = p->mesh->getDownward(lastTet, 2, lastTetFaces); - int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, firstFace); - int filast = apf::findIn(lastTetFaces, n_lastTetFaces, firstFace); - PCU_ALWAYS_ASSERT(fi1 != -1 && filast != -1); - - // first tet opp vertex crd - apf::MeshEntity* firstTetOppVert = getTetOppVert( - p->mesh, firstTet, firstFace); - apf::Vector3 firstTetOppVertCrd; - p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd); - - // last tet opp vertex crd - apf::MeshEntity* lastTetOppVert = getTetOppVert( - p->mesh, lastTet, firstFace); - apf::Vector3 lastTetOppVertCrd; - p->mesh->getPoint(lastTetOppVert, 0, lastTetOppVertCrd); - - // normal to the face - apf::Downward edge_vertices; - p->mesh->getDownward(p->entity, 0, edge_vertices); - apf::Vector3 p0, p1, p2; - p->mesh->getPoint(edge_vertices[0], 0, p0); - p->mesh->getPoint(edge_vertices[1], 0, p1); - apf::MeshEntity* faceOppVert = getFaceOppVert( - p->mesh, firstFace, p->entity); - p->mesh->getPoint(faceOppVert, 0, p2); - - apf::Vector3 normal = apf::cross(p1-p0, p2-p0); - - // direction vectors from p0 to opp tet verts - apf::Vector3 vFirst = firstTetOppVertCrd - p0; - apf::Vector3 vLast = lastTetOppVertCrd - p0; - - if ((vFirst * normal > 0) && (vLast * normal < 0)) { - // reverse list of tets and faces - std::reverse(p->tets.begin(), p->tets.end()); - std::reverse(p->faces.begin(), p->faces.begin()); - } - } -} - -static void runErm(EdgePatch* ep) -{ - getOrderedTetsandFaces(ep->mesh, ep->entity, ep->tets, ep->faces); - //getClockwiseTetsandFaces(ep); - assembleEdgePatchLHS(ep); - assembleEdgePatchRHS(ep); - mth::solveFromQR(ep->qr.Q, ep->qr.R, ep->b, ep->x); - - if (!ep->isOnBdry) { // solve At*mu = g for g - mth::Vector temp(ep->tets.size()); - mth::multiply(ep->At, ep->x, temp); - for (size_t i = 0; i < ep->tets.size(); i++) { - ep->x(i) = temp(i); - } - } - - int nf = ep->faces.size(); - for(int i = 0; i < nf; i++) { - apf::MeshEntity* face = ep->faces[i]; - - apf::Downward e; - int ned = ep->mesh->getDownward(face, 1, e); - int ei = apf::findIn(e, ned, ep->entity); - PCU_ALWAYS_ASSERT(ned == 3 && ei != -1); - - double components[3]; - apf::getComponents(ep->equilibration->g, face, 0, components); - components[ei] = ep->x(i); - apf::setComponents(ep->equilibration->g, face, 0, components); - } -} - - -class EdgePatchOp : public apf::CavityOp -{ -public: - EdgePatchOp(Equilibration* eq): - apf::CavityOp(eq->mesh) - { - setupEdgePatch(&edgePatch, eq); - } - virtual Outcome setEntity(apf::MeshEntity* e) - { - if (edgePatch.mesh->hasTag(e, edgePatch.equilibration->tag)) - return SKIP; - startEdgePatch(&edgePatch, e); - if ( ! buildEdgePatch(&edgePatch, this)) - return REQUEST; - return OK; - } - virtual void apply() - { - runErm(&edgePatch); - int n = VISITED; - edgePatch.mesh->setIntTag( - edgePatch.entity, edgePatch.equilibration->tag, &n); - } - EdgePatch edgePatch; -}; - -apf::Field* equilibrateResiduals(apf::Field* f) -{ - apf::Field* g = createPackedField( - apf::getMesh(f), "g", 3, apf::getConstant(2) ); - Equilibration equilibration; - setupEquilibration(&equilibration, f, g); - EdgePatchOp op(&equilibration); - op.applyToDimension(1); // edges - - apf::MeshEntity* ent; - apf::MeshIterator* it = apf::getMesh(f)->begin(1); - while ((ent = apf::getMesh(f)->iterate(it))) { - apf::getMesh(f)->removeTag(ent, equilibration.tag); - } - apf::getMesh(f)->end(it); - apf::getMesh(f)->destroyTag(equilibration.tag); - - return g; -} - - -} diff --git a/ree/reeSizeField.cc b/ree/reeSizeField.cc index 00765a01a..d2c483b15 100644 --- a/ree/reeSizeField.cc +++ b/ree/reeSizeField.cc @@ -156,14 +156,14 @@ apf::Field* getTargetEMSizeField( double alpha /*= 0.25*/, double beta /*= 2.0*/) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Sizefield sz; setupSizefield(&sz, ef, error_field, n, alpha, beta); getElementSizeField(&sz); averageSizeField(&sz); apf::destroyField(sz.element_size); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!ef->getMesh()->getPCU()->Self()) lion_eprint(1,"EM: SizeField computed in %f seconds\n",t1-t0); return sz.size; } From a2cb48fc72c066464fdc819b66cb18950ad13cad Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 25 Jan 2024 13:19:20 -0500 Subject: [PATCH 039/141] test folder fixes --- spr/sprEstimateError.cc | 4 +-- spr/sprEstimateTargetError.cc | 4 +-- stk/apfExodusOutput.cc | 4 +-- stk/apfSTK.cc | 2 +- test/cgns.cc | 50 +++++++++++++++++------------------ test/convert.cc | 12 ++++----- test/crack_test.cc | 4 +-- test/curvetest.cc | 4 +-- test/describe.cc | 2 +- test/dg_ma_test.cc | 2 +- test/elmBalance.cc | 2 +- test/fusion.cc | 2 +- test/fusion2.cc | 4 +-- 13 files changed, 48 insertions(+), 48 deletions(-) diff --git a/spr/sprEstimateError.cc b/spr/sprEstimateError.cc index 14aeaa592..72c68f618 100644 --- a/spr/sprEstimateError.cc +++ b/spr/sprEstimateError.cc @@ -311,11 +311,11 @@ static void estimateError(Estimation* e) apf::Field* getSPRSizeField(apf::Field* eps, double adaptRatio) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Estimation e; setupEstimation(&e, eps, adaptRatio); estimateError(&e); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!PCU_Comm_Self()) lion_eprint(1,"SPR: error estimated in %f seconds\n",t1-t0); return e.size; diff --git a/spr/sprEstimateTargetError.cc b/spr/sprEstimateTargetError.cc index 0e05f3420..0b29d5467 100644 --- a/spr/sprEstimateTargetError.cc +++ b/spr/sprEstimateTargetError.cc @@ -246,13 +246,13 @@ apf::Field* getTargetSPRSizeField( double alpha, double beta) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); PCU_ALWAYS_ASSERT(target > 0); PCU_ALWAYS_ASSERT(alpha < beta); target::Estimation e; target::setupEstimation(&e, eps, target, alpha, beta); target::estimateError(&e); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!PCU_Comm_Self()) lion_eprint(1, "SPR (target): error estimated in %f seconds\n",t1-t0); return e.vtx_size; diff --git a/stk/apfExodusOutput.cc b/stk/apfExodusOutput.cc index 24e24b184..f8cc560f8 100644 --- a/stk/apfExodusOutput.cc +++ b/stk/apfExodusOutput.cc @@ -53,14 +53,14 @@ void writeExodus( } if (bulk.is_null()) { - bulk = Teuchos::rcp(new stk::mesh::BulkData(*meta, PCU_Get_Comm())); + bulk = Teuchos::rcp(new stk::mesh::BulkData(*meta, mesh->getPCU()->GetMPIComm())); apf::copyMeshToBulk(n, models, meta.get(), bulk.get()); } apf::copyFieldsToBulk(n, meta.get(), bulk.get()); if (mesh_data.is_null()) { Ioss::Init::Initializer(); - mesh_data = Teuchos::rcp(new stk::io::StkMeshIoBroker(PCU_Get_Comm())); + mesh_data = Teuchos::rcp(new stk::io::StkMeshIoBroker(mesh->getPCU()->GetMPIComm())); output_file_idx = mesh_data->create_output_mesh(filename, stk::io::WRITE_RESULTS); mesh_data->set_bulk_data(*bulk); diff --git a/stk/apfSTK.cc b/stk/apfSTK.cc index d0dc5d853..1182e2ca9 100644 --- a/stk/apfSTK.cc +++ b/stk/apfSTK.cc @@ -769,7 +769,7 @@ long getStkId(GlobalNumbering* numbers, Node node) StkModels* create_sets(Mesh* m, const char* filename) { StkModels* sets = new StkModels; - if (! PCU_Comm_Self()) + if (! m->getPCU()->Self()) lion_oprint(1,"reading association file: %s\n", filename); static std::string const setNames[3] = { "node set", "side set", "elem set"}; diff --git a/test/cgns.cc b/test/cgns.cc index 6f853d2e0..892d5e515 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -24,7 +24,7 @@ apf::MeshTag *create_int_tag(const std::string &name, apf::Mesh2 *m, int dim) apf::MeshEntity *elem = nullptr; apf::MeshIterator *it = m->begin(dim); int vals[1]; - vals[0] = PCU_Comm_Self(); + vals[0] = m->getPCU()->Self(); while ((elem = m->iterate(it))) m->setIntTag(elem, tag, vals); m->end(it); @@ -68,7 +68,7 @@ void balance(const std::string &prefix, const apf::ZoltanMethod &method, apf::Me Parma_PrintPtnStats(m, ""); m->destroyTag(weights); - const std::string name = prefix + "_balance_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_balance_" + std::to_string(m->getPCU()->Peers()) + "procs"; apf::writeVtkFiles(name.c_str(), m); } @@ -103,7 +103,7 @@ void simpleReorder(const std::string &prefix, apf::Mesh2 *m) apf::makeGlobal(apf::numberElements(m, "elem Idx_pstOrder")); } - const std::string name = prefix + "_reorder_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_reorder_" + std::to_string(m->getPCU()->Peers()) + "procs"; apf::writeVtkFiles(name.c_str(), m); } @@ -154,7 +154,7 @@ auto additional(const std::string &prefix, gmi_model *g, apf::Mesh2 *mesh) } pm->end(it); - const std::string name = prefix + "_toPUMI_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_toPUMI_" + std::to_string(mesh->getPCU()->Peers()) + "procs"; // only the owned elements will have a elmField value of 1 pumi_mesh_write(pm, (name + "_beforeSync").c_str(), "vtk"); @@ -200,45 +200,45 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: if (dim == 3) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh"); apf::writeVtkFiles(name.c_str(), m, 3); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_faceMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_faceMesh"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 2) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 1) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh"); apf::writeVtkFiles(name.c_str(), m, 0); } } @@ -246,7 +246,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: std::string cgnsOutputName; if (writeCGNS) { - cgnsOutputName = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + "_outputFile.cgns"; + cgnsOutputName = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + "_outputFile.cgns"; apf::writeCGNS(cgnsOutputName.c_str(), m, cgnsBCMap); } @@ -386,45 +386,45 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: if (dim == 3) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 3); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_faceMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_faceMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 2) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 1) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 0); } } @@ -432,7 +432,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: if (writeCGNS) { // what this one to be re-read if doing re-reading so that the dummy variables (vector/matrix/scalar) are there - cgnsOutputName = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + "_additional_outputFile.cgns"; + cgnsOutputName = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + "_additional_outputFile.cgns"; apf::writeCGNS(cgnsOutputName.c_str(), m, cgnsBCMap); } // @@ -454,7 +454,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: apf::displaceMesh(m, field); if (writeCGNS) { - std::string cgnsOutputName = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + "_additional_moved_outputFile.cgns"; + std::string cgnsOutputName = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + "_additional_moved_outputFile.cgns"; apf::writeCGNS(cgnsOutputName.c_str(), m, cgnsBCMap); } } diff --git a/test/convert.cc b/test/convert.cc index 904c0797a..946ed7feb 100644 --- a/test/convert.cc +++ b/test/convert.cc @@ -377,7 +377,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c faces=PList_new(); err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); PList_delete(regions); // not used so delete - if(err!=1 && !PCU_Comm_Self()) + if(err!=1 && !simApfMesh->getPCU()->Self()) fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err); // for each face in the returned list of faces @@ -478,9 +478,9 @@ int main(int argc, char** argv) assert(false); */ - double t0 = PCU_Time(); + double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if(!PCU_Comm_Self()) fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0); @@ -488,14 +488,14 @@ int main(int argc, char** argv) addFathersTag(simModel, sim_mesh, simApfMesh, extruRootPath); - double t2 = PCU_Time(); + double t2 = pcu::Time(); if(!PCU_Comm_Self()) fprintf(stderr, "created the apf_sim mesh in %f seconds\n", t2-t1); if (should_attach_order) attachOrder(simApfMesh); apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh); - double t3 = PCU_Time(); - if(!PCU_Comm_Self()) + double t3 = pcu::Time(); + if(!mesh->getPCU()->Self()) fprintf(stderr, "created the apf_mds mesh in %f seconds\n", t3-t2); apf::printStats(mesh); diff --git a/test/crack_test.cc b/test/crack_test.cc index 148735a3f..879770c0c 100644 --- a/test/crack_test.cc +++ b/test/crack_test.cc @@ -28,12 +28,12 @@ void bCurver(const char* modelFile, const char* meshFile, { apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("attempting to curve the mesh to %d order Bezier\n", order); crv::BezierCurver bc(m, order, blendOrder); bc.run(); m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("succeeded!\n"); crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, order + 2, outputPrefix); diff --git a/test/curvetest.cc b/test/curvetest.cc index 3bf962ec2..1c1f7585d 100644 --- a/test/curvetest.cc +++ b/test/curvetest.cc @@ -77,8 +77,8 @@ static void testElementSize(apf::Mesh* m) m->end(it); } - PCU_Add_Doubles(&(sizes[0]),3); - if(!PCU_Comm_Self()) + m->getPCU()->Add(&(sizes[0]),3); + if(!m->getPCU()->Self()) printf("Total sizes for order %d %f %f %f\n", m->getShape()->getOrder(), sizes[0],sizes[1],sizes[2]); diff --git a/test/describe.cc b/test/describe.cc index c4a141c5b..c7171f287 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -85,7 +85,7 @@ static double get_chunks() static void list_tags(apf::Mesh* m) { - if (PCU_Comm_Self()) + if (m->getPCU()->Self()) return; apf::DynamicArray tags; m->getTags(tags); diff --git a/test/dg_ma_test.cc b/test/dg_ma_test.cc index 188592c6a..c29ff5ce1 100644 --- a/test/dg_ma_test.cc +++ b/test/dg_ma_test.cc @@ -55,7 +55,7 @@ int main(int argc, char** argv) m->verify(); Linear sf(m); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) printf("Matched mesh: disabling" " snapping, and shape correction,\n"); in->shouldSnap = false; diff --git a/test/elmBalance.cc b/test/elmBalance.cc index 059c18657..5ce81d5d7 100644 --- a/test/elmBalance.cc +++ b/test/elmBalance.cc @@ -48,7 +48,7 @@ int main(int argc, char** argv) apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); - if(!PCU_Comm_Self()) + if(!m->getPCU()->Self()) fprintf(stdout, "imbalance %.3f %.3f %.3f %.3f\n", imbalance[0], imbalance[1], imbalance[2], imbalance[3]); apf::MeshTag* weights = setWeights(m); diff --git a/test/fusion.cc b/test/fusion.cc index 7da4c72f2..582188417 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -161,7 +161,7 @@ int main( int argc, char* argv[]) GroupCode code; code.model = makeModel(); code.meshFile = argv[1]; - apf::Unmodulo outMap(PCU_Comm_Self(), 2); + apf::Unmodulo outMap(code.mesh->getPCU()->Self(), 2); Parma_SplitPartition(NULL, 2, code); apf::remapPartition(code.mesh, outMap); code.mesh->verify(); diff --git a/test/fusion2.cc b/test/fusion2.cc index eef2b6c10..cd06fe2ca 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -68,7 +68,7 @@ static void backToTags(apf::Mesh2* m) static void connectPlanes(apf::Mesh2* m) { /* dont use any of this function in production. */ - int peer = 1 - PCU_Comm_Self(); + int peer = 1 - m->getPCU()->Self(); for (int i = 0; i < 3; ++i) { apf::MeshEntity* v = apf::getMdsEntity(m, 0, i); m->addRemote(v, peer, v); //pointers aren't usually equal @@ -108,7 +108,7 @@ struct GroupCode : public Parma_GroupCode static void copyPlane(apf::Mesh2* m) { /* this mimics the copy entity construction */ - if (PCU_Comm_Self() == 1) + if (m->getPCU()->Self() == 1) addOneTri(m); /* this connects the two one-triangle planes together */ connectPlanes(m); From da4383dbe8a547817a6f4d2d97fbb6205fb653ad Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 25 Jan 2024 13:32:19 -0500 Subject: [PATCH 040/141] More test folder fixes --- test/H1Shapes.cc | 6 +++--- test/L2Shapes.cc | 12 ++++++------ test/generate.cc | 2 +- test/intrude.cc | 2 +- test/matchedNodeElmReader.cc | 10 +++++----- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc index d69b978f1..f65899074 100644 --- a/test/H1Shapes.cc +++ b/test/H1Shapes.cc @@ -140,12 +140,12 @@ void testH1( for (int d = 0; d <= dim; d++) { if (!h1Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "no nodes in dimension %d\n", d); continue; } else - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "computing dofs for dimension %d\n", d); it = m->begin(d); while( (ent = m->iterate(it)) ) { @@ -210,7 +210,7 @@ void testH1( m->end(it); // check for field interpolation - if(0==PCU_Comm_Self()) { + if(0==m->getPCU()->Self()) { lion_oprint(1, "L2Error for entities of dimension %d is %e\n", d, L2Error); } PCU_ALWAYS_ASSERT_VERBOSE(L2Error < 1.e-12, diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc index a2db1e071..d8af11975 100644 --- a/test/L2Shapes.cc +++ b/test/L2Shapes.cc @@ -96,12 +96,12 @@ void testL2( for (int d = 0; d <= dim; d++) { if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "no nodes in dimension %d\n", d); continue; } else - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "computing dofs for dimension %d\n", d); it = m->begin(d); while( (ent = m->iterate(it)) ) { @@ -147,7 +147,7 @@ void testL2( m->end(it); // check for field interpolation - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE); PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16, "Fields were not interpolated correctly!"); @@ -169,12 +169,12 @@ void testL2writeNative( for (int d = 0; d <= dim; d++) { if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "no nodes in dimension %d\n", d); continue; } else - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "computing dofs for dimension %d\n", d); it = m->begin(d); while( (ent = m->iterate(it)) ) { @@ -207,7 +207,7 @@ void testL2writeNative( apf::Mesh2* m2 = apf::loadMdsMesh(".null", "./L2Shape_test_mesh.smb"); int fCount = 0; for (int i = 0; i < m2->countFields(); i++) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "field %d's name and shape are %s and %s\n", i, m2->getField(i)->getName(), m2->getField(i)->getShape()->getName()); fCount++; diff --git a/test/generate.cc b/test/generate.cc index a5c94f15e..523edf3b5 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -318,7 +318,7 @@ int main(int argc, char** argv) MPI_Init(&argc, &argv); PCU_Comm_Init(); lion_set_verbosity(1); - PCU_Protect(); + pcu::Protect(); getConfig(argc,argv); if (should_attach_order && should_fix_pyramids) { diff --git a/test/intrude.cc b/test/intrude.cc index f5bc2f893..3e5fb6cf2 100644 --- a/test/intrude.cc +++ b/test/intrude.cc @@ -36,7 +36,7 @@ int main(int argc, char** argv) m->findModelEntity(0, 1))); size_t nlayers; ma::intrude(m, extrusions, &nlayers); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) std::cout << "counted " << nlayers << " layers\n"; m->writeNative(argv[3]); m->destroyNative(); diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index 4dfdfabd0..d0ae8b1d1 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -424,7 +424,7 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) { } - + void getLocalRange(apf::Gid total, int& local, apf::Gid& first, apf::Gid& last) { const int self = PCU_Comm_Self(); @@ -769,7 +769,7 @@ int main(int argc, char** argv) if( argc == 11 ) noVerify=atoi(argv[10]); - double t0 = PCU_Time(); + double t0 = pcu::Time(); MeshInfo m; readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m); @@ -807,7 +807,7 @@ int main(int argc, char** argv) if( strcmp(argv[6], "NULL") ) { setMappedTag(mesh, "fathers2D", m.fathers2D, 1, m.localNumVerts, outMap); delete [] m.fathers2D; - } else if(!PCU_Comm_Self()) + } else if(!mesh->getPCU()->Self()) fprintf(stderr, "fathers2D not requested \n"); if(0==1) { @@ -816,8 +816,8 @@ int main(int argc, char** argv) (void) ts; } - if(!PCU_Comm_Self()) - fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0); + if(!mesh->getPCU()->Self()) + fprintf(stderr, "seconds to create mesh %.3f\n", pcu::Time()-t0); if(noVerify != 1) mesh->verify(); outMap.clear(); From 3c7b4479a3c320ff62e9ce17b78b0ec983f23696 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 26 Jan 2024 16:31:10 -0500 Subject: [PATCH 041/141] fixed last of test folder and made PCU2 .h and .cc --- pcu/PCU2.h | 125 +++++++++ pcu/pcu2.cc | 518 +++++++++++++++++++++++++++++++++++ test/measureAnisoStats.cc | 8 +- test/measureIsoStats.cc | 8 +- test/ph_adapt.cc | 2 +- test/pumi.cc | 10 +- test/quality.cc | 10 +- test/rm_extrusion.cc | 2 +- test/runSimxAnisoAdapt.cc | 8 +- test/serialize.cc | 2 +- test/sim_part.cc | 4 +- test/visualizeAnisoSizes.cc | 2 +- test/vtxEdgeElmBalance.cc | 4 +- test/writePart.cc | 2 +- zoltan/apfInterElement.cc | 12 +- zoltan/apfZoltan.cc | 16 +- zoltan/apfZoltanCallbacks.cc | 6 +- 17 files changed, 691 insertions(+), 48 deletions(-) create mode 100644 pcu/PCU2.h create mode 100644 pcu/pcu2.cc diff --git a/pcu/PCU2.h b/pcu/PCU2.h new file mode 100644 index 000000000..ac62ec10b --- /dev/null +++ b/pcu/PCU2.h @@ -0,0 +1,125 @@ +#ifndef PCU2_H +#define PCU2_H +#include "pcu_defines.h" +#include + +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif + +int PCU_Comm_Init2(PCUHandle& h); +int PCU_Comm_Free2(PCUHandle& h); + +/*rank/size functions*/ +int PCU_Comm_Self2(PCUHandle h); +int PCU_Comm_Peers2(PCUHandle h); + +/*recommended message passing API*/ +void PCU_Comm_Begin2(PCUHandle h); +int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void* data, size_t size); +#define PCU_COMM_PACK2(to_rank,object)\ +PCU_Comm_Pack2(to_rank,&(object),sizeof(object)) +int PCU_Comm_Send2(PCUHandle h); +bool PCU_Comm_Receive2(PCUHandle h); +bool PCU_Comm_Listen2(PCUHandle h); +int PCU_Comm_Sender2(PCUHandle h); +bool PCU_Comm_Unpacked2(PCUHandle h); +int PCU_Comm_Unpack2(PCUHandle h, void* data, size_t size); +#define PCU_COMM_UNPACK(object)\ +PCU_Comm_Unpack2(&(object),sizeof(object)) + +/*turns deterministic ordering for the + above API on/off*/ +void PCU_Comm_Order2(PCUHandle h, bool on); + +/*collective operations*/ +void PCU_Barrier2(PCUHandle h); +void PCU_Add_Doubles2(PCUHandle h, double* p, size_t n); +double PCU_Add_Double2(PCUHandle h, double x); +void PCU_Min_Doubles2(PCUHandle h, double* p, size_t n); +double PCU_Min_Double2(PCUHandle h, double x); +void PCU_Max_Doubles2(PCUHandle h, double* p, size_t n); +double PCU_Max_Double2(PCUHandle h, double x); +void PCU_Add_Ints2(PCUHandle h, int* p, size_t n); +int PCU_Add_Int2(PCUHandle h, int x); +void PCU_Add_Longs2(PCUHandle h, long* p, size_t n); +long PCU_Add_Long2(PCUHandle h, long x); +void PCU_Exscan_Ints2(PCUHandle h, int* p, size_t n); +int PCU_Exscan_Int2(PCUHandle h, int x); +void PCU_Exscan_Longs2(PCUHandle h, long* p, size_t n); +long PCU_Exscan_Long2(PCUHandle h, long x); +void PCU_Add_SizeTs2(PCUHandle h, size_t* p, size_t n); +size_t PCU_Add_SizeT2(PCUHandle h, size_t x); +void PCU_Min_SizeTs2(PCUHandle h, size_t* p, size_t n); +size_t PCU_Min_SizeT2(PCUHandle h, size_t x); +void PCU_Max_SizeTs2(PCUHandle h, size_t* p, size_t n); +size_t PCU_Max_SizeT2(PCUHandle h, size_t x); +void PCU_Min_Ints2(PCUHandle h, int* p, size_t n); +int PCU_Min_Int2(PCUHandle h, int x); +void PCU_Max_Ints2(PCUHandle h, int* p, size_t n); +int PCU_Max_Int2(PCUHandle h, int x); +void PCU_Max_Longs2(PCUHandle h, long* p, size_t n); +long PCU_Max_Long2(PCUHandle h, long x); +int PCU_Or2(PCUHandle h, int c); +int PCU_And2(PCUHandle h, int c); + +/*process-level self/peers (mpi wrappers)*/ +int PCU_Proc_Self2(PCUHandle h); +int PCU_Proc_Peers2(PCUHandle h); + +/*IPComMan replacement API*/ +int PCU_Comm_Write2(PCUHandle h, int to_rank, const void* data, size_t size); +#define PCU_COMM_WRITE(to,data) \ +PCU_Comm_Write2(to,&(data),sizeof(data)) +bool PCU_Comm_Read2(PCUHandle h, int* from_rank, void** data, size_t* size); + +/*Debug file I/O API*/ +void PCU_Debug_Open(PCUHandle h); + +void PCU_Debug_Print2(PCUHandle h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(1,2); +/*lesser-used APIs*/ +bool PCU_Comm_Initialized2(PCUHandle h); +int PCU_Comm_Packed2(PCUHandle h ,int to_rank, size_t* size); +int PCU_Comm_From2(PCUHandle h, int* from_rank); +int PCU_Comm_Received2(PCUHandle h, size_t* size); +void* PCU_Comm_Extract2(PCUHandle h, size_t size); +int PCU_Comm_Rank2(PCUHandle h, int* rank); +int PCU_Comm_Size2(PCUHandle h, int* size); + +/*deprecated method enum*/ +#ifdef __cplusplus +enum PCU_Method { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD }; +#else +typedef enum { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD } PCU_Method; +#endif +int PCU_Comm_Start2(PCUHandle h, PCU_Method method); + +/*special MPI_Comm replacement API*/ +void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm); +MPI_Comm PCU_Get_Comm2(PCUHandle h); + +/*stack trace helpers using GNU/Linux*/ +void PCU_Protect2(void); + +/*MPI_Wtime() equivalent*/ +double PCU_Time2(void); + +/*Memory usage*/ +double PCU_GetMem2(void); + +/*Access global variable*/ + + + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif \ No newline at end of file diff --git a/pcu/pcu2.cc b/pcu/pcu2.cc new file mode 100644 index 000000000..fdb79c7a9 --- /dev/null +++ b/pcu/pcu2.cc @@ -0,0 +1,518 @@ +#include "PCU2.h" +#include "PCUObj.h" +#include "reel.h" +#include + +struct PCUHandle { + void* ptr +}; + +extern "C" { + +int PCU_Comm_Init2(PCUHandle& h) { + if (h->ptr != nullptr) + reel_fail("nested calls to Comm_Init"); + pcu::PCU* pcu_object = new pcu::PCU(MPI_COMM_WORLD); + //PCUHandle h; + h->ptr = static_cast(pcu_object); + return PCU_SUCCESS; +} + +int PCU_Comm_Free2(PCUHandle& h) { + if (h->ptr == nullptr) + reel_fail("Comm_Free called before Comm_Init"); + delete h->ptr; + h->ptr = nullptr; + return PCU_SUCCESS; +} + +int PCU_Comm_Self2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Self called before Comm_Init"); + return static_cast(h->ptr)->Self(); +} + +int PCU_Comm_Peers2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Peers called before Comm_Init"); + return static_cast(h->ptr)->Peers(); +} + +void PCU_Comm_Begin2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Begin called before Comm_Init"); + static_cast(h->ptr)->Begin(); +} + +/** \brief Packs data to be sent to \a to_rank. + \details This function appends the block of \a size bytes starting + at \a data to the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void *data, size_t size) { + if (h->ptr == nullptr) + reel_fail("Comm_Pack called before Comm_Init"); + return static_cast(h->ptr)->Pack(to_rank, data, size); +} + +/** \brief Sends all buffers for this communication phase. + \details This function should be called by all threads in the MPI job + after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls + to PCU_Comm_Listen or PCU_Comm_Read. + All buffers from this thread are sent out and receiving + may begin after this call. + */ +int PCU_Comm_Send2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Send called before Comm_Init"); + return static_cast(h->ptr)->Send(); +} + +/** \brief Tries to receive a buffer for this communication phase. + \details Either this function or PCU_Comm_Read should be called at least + once by all threads during the communication phase, after PCU_Comm_Send + is called. The result will be false if and only if the communication phase + is over and there are no more buffers to receive. + Otherwise, a buffer was received. + Its contents are retrievable through PCU_Comm_Unpack, and its metadata through + PCU_Comm_Sender and PCU_Comm_Received. + Users should unpack all data from this buffer before calling this function + again, because the previously received buffer is destroyed by the call. + */ +bool PCU_Comm_Listen2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Listen called before Comm_Init"); + return static_cast(h->ptr)->Listen(); +} + +/** \brief Returns in * \a from_rank the sender of the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + */ +int PCU_Comm_Sender2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Sender called before Comm_Init"); + return static_cast(h->ptr)->Sender(); +} + +bool PCU_Comm_Unpacked2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Unpacked called before Comm_Init"); + return static_cast(h->ptr)->Unpacked(); +} + +/** \brief Unpacks a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + \a data must point to a block of memory of at least \a size bytes, into + which the next \a size bytes of the current received buffer will be written. + Subsequent calls will begin unpacking where this call left off, + so that the entire received buffer can be unpacked by a sequence of calls to + this function. + Users must ensure that there remain \a size bytes to be unpacked, + PCU_Comm_Unpacked can help with this. + */ +int PCU_Comm_Unpack2(PCUHandle h, void *data, size_t size) { + if (h->ptr == nullptr) + reel_fail("Comm_Unpack called before Comm_Init"); + return static_cast(h->ptr)->Unpack(data, size); +} + +void PCU_Comm_Order2(PCUHandle h, bool on) { + if (h->ptr == nullptr) + reel_fail("Comm_Order called before Comm_Init"); + static_cast(h->ptr)->Order(on); +} + +/** \brief Blocking barrier over all threads. */ +void PCU_Barrier2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Barrier called before Comm_Init"); + static_cast(h->ptr)->Barrier(); +} + +/** \brief Performs an Allreduce sum of double arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n doubles. + After this call, p[i] will contain the sum of all p[i]'s + given by each rank. + */ +void PCU_Add_Doubles2(PCUHandle h, double *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Add_Doubles called before Comm_Init"); + static_cast(h->ptr)->Add(p, n); +} + +double PCU_Add_Double2(PCUHandle h, double x) { + if (h->ptr == nullptr) + reel_fail("Add_Double called before Comm_Init"); + return static_cast(h->ptr)->Add(x); +} + +/** \brief Performs an Allreduce minimum of double arrays. + */ +void PCU_Min_Doubles2(PCUHandle h, double *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Min_Doubles called before Comm_Init"); + static_cast(h->ptr)->Min(p, n); +} + +double PCU_Min_Double2(PCUHandle h, double x) { + if (h->ptr == nullptr) + reel_fail("Min_Double called before Comm_Init"); + return static_cast(h->ptr)->Min(x); +} + +/** \brief Performs an Allreduce maximum of double arrays. + */ +void PCU_Max_Doubles2(PCUHandle, double *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Max_Doubles called before Comm_Init"); + static_cast(h->ptr)->Max(p, n); +} + +double PCU_Max_Double2(PCUHandle h, double x) { + if (h->ptr == nullptr) + reel_fail("Max_Double called before Comm_Init"); + return static_cast(h->ptr)->Max(x); +} + +/** \brief Performs an Allreduce sum of integers + */ +void PCU_Add_Ints2(PCUHandle h, int *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Add_Ints called before Comm_Init"); + static_cast(h->ptr)->Add(p, n); +} + +int PCU_Add_Int2(PCUHandle h, int x) { + if (h->ptr == nullptr) + reel_fail("Add_Int called before Comm_Init"); + return static_cast(h->ptr)->Add(x); +} + +/** \brief Performs an Allreduce sum of long integers + */ +void PCU_Add_Longs2(PCUHandle h, long *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Add_Longs called before Comm_Init"); + static_cast(h->ptr)->Add(p, n); +} + +long PCU_Add_Long2(PCUHandle h, long x) { + if (h->ptr == nullptr) + reel_fail("Add_Long called before Comm_Init"); + return static_cast(h->ptr)->Add(x); +} + +/** \brief Performs an Allreduce sum of size_t unsigned integers + */ +void PCU_Add_SizeTs2(PCUHandle h, size_t *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Add_SizeTs called before Comm_Init"); + static_cast(h->ptr)->Add(p, n); +} + +size_t PCU_Add_SizeT2(PCUHandle h, size_t x) { + if (h->ptr == nullptr) + reel_fail("Add_SizeT called before Comm_Init"); + return static_cast(h->ptr)->Add(x); +} + +/** \brief Performs an Allreduce minimum of size_t unsigned integers + */ +void PCU_Min_SizeTs2(PCUHandle h, size_t *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Min_SizeTs called before Comm_Init"); + static_cast(h->ptr)->Min(p, n); +} + +size_t PCU_Min_SizeT2(PCUHandle h, size_t x) { + if (h->ptr == nullptr) + reel_fail("Min_SizeT called before Comm_Init"); + return static_cast(h->ptr)->Min(x); +} + +/** \brief Performs an Allreduce maximum of size_t unsigned integers + */ +void PCU_Max_SizeTs2(PCUHandle h, size_t *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Max_SizeTs called before Comm_Init"); + static_cast(h->ptr)->Max(p, n); +} + +size_t PCU_Max_SizeT2(PCUHandle h, size_t x) { + if (h->ptr == nullptr) + reel_fail("Max_SizeT called before Comm_Init"); + return static_cast(h->ptr)->Max(x); +} + +/** \brief Performs an exclusive prefix sum of integer arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n integers. + After this call, p[i] will contain the sum of all p[i]'s + given by ranks lower than the calling rank. + */ +void PCU_Exscan_Ints2(PCUHandle h, int *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Exscan_Ints called before Comm_Init"); + static_cast(h->ptr)->Exscan(p, n); +} + +int PCU_Exscan_Int2(PCUHandle h, int x) { + if (h->ptr == nullptr) + reel_fail("Exscan_Int called before Comm_Init"); + return static_cast(h->ptr)->Exscan(x); +} + +/** \brief See PCU_Exscan_Ints */ +void PCU_Exscan_Longs2(PCUHandle h, long *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Exscan_Longs called before Comm_Init"); + static_cast(h->ptr)->Exscan(p, n); +} + +long PCU_Exscan_Long2(PCUHandle h, long x) { + if (h->ptr == nullptr) + reel_fail("Exscan_Long called before Comm_Init"); + return static_cast(h->ptr)->Exscan(x); +} + +/** \brief Performs an Allreduce minimum of int arrays. + */ +void PCU_Min_Ints2(PCUHandle h, int *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Min_Ints called before Comm_Init"); + static_cast(h->ptr)->Min(p, n); +} + +int PCU_Min_Int2(PCUHandle h, int x) { + if (h->ptr == nullptr) + reel_fail("Min_Int called before Comm_Init"); + return static_cast(h->ptr)->Min(x); +} + +/** \brief Performs an Allreduce maximum of int arrays. + */ +void PCU_Max_Ints2(PCUHandle h, int *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Max_Ints called before Comm_Init"); + global_pcu->Max(p, n); +} + +int PCU_Max_Int2(PCUHandle h, int x) { + if (h->ptr == nullptr) + reel_fail("Max_Int called before Comm_Init"); + return static_cast(h->ptr)->Max(x); +} +/** \brief Performs an Allreduce maximum of long arrays. + */ +void PCU_Max_Longs2(PCUHandle h, long *p, size_t n) { + if (h->ptr == nullptr) + reel_fail("Max_Longs called before Comm_Init"); + static_cast(h->ptr)->Max(p, n); +} + +long PCU_Max_Long2(PCUHandle h, long x) { + if (h->ptr == nullptr) + reel_fail("Max_Long called before Comm_Init"); + return static_cast(h->ptr)->Max(x); +} + +/** \brief Performs a parallel logical OR reduction + */ +int PCU_Or2(PCUHandle h, int c) { + if (h->ptr == nullptr) + reel_fail("Or called before Comm_Init"); + return static_cast(h->ptr)->Or(c); +} + +/** \brief Performs a parallel logical AND reduction + */ +int PCU_And2(PCUHandle h, int c) { + if (h->ptr == nullptr) + reel_fail("And called before Comm_Init"); + return static_cast(h->ptr)->And(c); +} + +/** \brief Returns the unique rank of the calling process. + */ +int PCU_Proc_Self2(PCUHandle h, void) { + if (h->ptr == nullptr) + reel_fail("Proc_Self called before Comm_Init"); + return static_cast(h->ptr)->Self(); +} + +/** \brief Returns the number of processes. + */ +int PCU_Proc_Peers2(PCUHandle h, void) { + if (h->ptr == nullptr) + reel_fail("Proc_Peers called before Comm_Init"); + return static_cast(h->ptr)->Peers(); +} + +/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. + */ +int PCU_Comm_Rank2(PCUHandle h, int *rank) { + if (h->ptr == nullptr) + reel_fail("Comm_Rank called before Comm_Init"); + *rank = static_cast(h->ptr)->Self(); + return PCU_SUCCESS; +} + +/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ +int PCU_Comm_Size2(PCUHandle h, int *size) { + if (h->ptr == nullptr) + reel_fail("Comm_Size called before Comm_Init"); + *size = static_cast(h->ptr)->Peers(); + return PCU_SUCCESS; +} + +/** \brief Returns true iff PCU has been initialized */ +bool PCU_Comm_Initialized2(PCUHandle h) { return h->ptr != nullptr; } + +/** \brief Deprecated, see PCU_Comm_Begin. + */ +int PCU_Comm_Start2(PCUHandle h, PCU_Method method) { + (void)method; // warning silencer + static_cast(h->ptr)->Begin(); + return PCU_SUCCESS; +} + +/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. + \details Returns the size of the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Packed2(PCUHandle h, int to_rank, size_t *size) { + if (h->ptr == nullptr) + reel_fail("Comm_Packed called before Comm_Init"); + return static_cast(h->ptr)->Packed(to_rank, size); +} + +/** \brief Packs a message to be sent to \a to_rank. + \details This function packs a message into the buffer being sent + to \a to_rank. + Messages packed by this function can be received using the function + PCU_Comm_Read. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + If this function is used, PCU_Comm_Pack should not be used. + */ +int PCU_Comm_Write2(PCUHandle h, int to_rank, const void *data, size_t size) { + if (h->ptr == nullptr) + reel_fail("Comm_Write called before Comm_Init"); + return static_cast(h->ptr)->Write(to_rank, data, size); +} + +/** \brief Convenience wrapper over Listen and Unpacked */ +bool PCU_Comm_Receive2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Comm_Receive called before Comm_Init"); + return static_cast(h->ptr)->Receive(); +} + +/** \brief Receives a message for this communication phase. + \details This function tries to receive a message packed by + PCU_Comm_Write. + If a the communication phase is over and there are no more + messages to receive, this function returns false. + Otherwise, * \a from_rank will be the rank which sent the message, + * \a data will point to the start of the message data, and + * \a size will be the number of bytes of message data. + If this function is used, PCU_Comm_Receive should not be used. + Note that the address * \a data points into a PCU buffer, so + it is strongly recommended that this data be read and not modified. + */ +bool PCU_Comm_Read2(PCUHandle h, int *from_rank, void **data, size_t *size) { + if (h->ptr == nullptr) + reel_fail("Comm_Read called before Comm_Init"); + return static_cast(h->ptr)->Read(from_rank, data, size); +} + +void PCU_Debug_Open2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Debug_Open called before Comm_Init"); + static_cast(h->ptr)->DebugOpen(); +} + +/** \brief like fprintf, contents go to debugN.txt */ +void PCU_Debug_Print2(PCUHandle h, const char *format, ...) { + if (h->ptr == nullptr) + reel_fail("Debug_Print called before Comm_Init"); + va_list arglist; + va_start(arglist, format); + static_cast(h->ptr)->DebugPrint(format, arglist); + va_end(arglist); +} + +/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ +int PCU_Comm_From2(PCUHandle h, int *from_rank) { + if (h->ptr == nullptr) + reel_fail("Comm_From called before Comm_Init"); + return static_cast(h->ptr)->From(from_rank); +} + +/** \brief Returns in * \a size the bytes in the current received buffer + \details This function should be called after a successful PCU_Comm_Receive. + The size returned will be the total received size regardless of how + much unpacking has been done. + */ +int PCU_Comm_Received2(PCUHandle h, size_t *size) { + if (h->ptr == nullptr) + reel_fail("Comm_Received called before Comm_Init"); + return static_cast(h->ptr)->Received(size); +} + +/** \brief Extracts a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Receive. + The next \a size bytes of the current received buffer are unpacked, + and an internal pointer to that data is returned. + The returned pointer must not be freed by the user. + */ +void *PCU_Comm_Extract2(PCUHandle h, size_t size) { + if (h->ptr == nullptr) + reel_fail("Comm_Extract called before Comm_Init"); + return static_cast(h->ptr)->Extract(size); +} + +/** \brief Reinitializes PCU with a new MPI communicator. + \details All of PCU's logic is based off two duplicates + of this communicator, so you can safely get PCU to act + on sub-groups of processes using this function. + This call should be collective over all processes + in the previous communicator. This is a very heavy weight function + and should be used sparingly. + */ +void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm) { + if (h->ptr == nullptr) + reel_fail("Switch_Comm called before Comm_Init"); + static_cast(h->ptr)->SwitchMPIComm(new_comm); +} + +/** \brief Return the current MPI communicator + \details Returns the communicator given to the + most recent PCU_Switch_Comm call, or MPI_COMM_WORLD + otherwise. + */ +MPI_Comm PCU_Get_Comm2(PCUHandle h) { + if (h->ptr == nullptr) + reel_fail("Get_Comm called before Comm_Init"); + return static_cast(h->ptr)->GetMPIComm(); +} + +/** \brief Return the time in seconds since some time in the past + */ +double PCU_Time2(void) { return pcu::Time(); } + +void PCU_Protect2(void) { return pcu::Protect(); } + +double PCU_GetMem2(void) { return pcu::GetMem(); } + + +/** \brief Return the global PCU + */ +pcu::PCU* PCU_GetGlobal2(PCUHandle h) { return static_cast(h->ptr); } + + + +} \ No newline at end of file diff --git a/test/measureAnisoStats.cc b/test/measureAnisoStats.cc index 4b67c380f..a3d16e968 100644 --- a/test/measureAnisoStats.cc +++ b/test/measureAnisoStats.cc @@ -182,7 +182,7 @@ void getStats( m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("\n evaluating the statistics! \n"); // get the stats ma::SizeField* sf = ma::makeSizeField(m, sizes, frames, true); @@ -235,8 +235,8 @@ void getStats( const char* pathName = tmp.c_str(); safe_mkdir(pathName); - ssq << pathName << "/linearQTable_" << PCU_Comm_Self() << ".dat"; - sse << pathName << "/linearETable_" << PCU_Comm_Self() << ".dat"; + ssq << pathName << "/linearQTable_" << m->getPCU()->Self() << ".dat"; + sse << pathName << "/linearETable_" << m->getPCU()->Self() << ".dat"; ssm << pathName << "/mesh_quality_vis"; ssl << pathName << "/mesh_edge_length_vis"; @@ -300,7 +300,7 @@ void getStats( } std::stringstream sstq; - sstq << pathName << "/linearBLTriQTable_" << PCU_Comm_Self() << ".dat"; + sstq << pathName << "/linearBLTriQTable_" << m->getPCU()->Self() << ".dat"; writeTable(sstq.str().c_str(), tri_qtable); } #endif diff --git a/test/measureIsoStats.cc b/test/measureIsoStats.cc index ac523843a..9d9e430e8 100644 --- a/test/measureIsoStats.cc +++ b/test/measureIsoStats.cc @@ -173,7 +173,7 @@ void getStats( m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("\n evaluating the statistics! \n"); // get the stats ma::SizeField* sf = ma::makeSizeField(m, sizes); @@ -226,8 +226,8 @@ void getStats( const char* pathName = tmp.c_str(); safe_mkdir(pathName); - ssq << pathName << "/linearQTable_" << PCU_Comm_Self() << ".dat"; - sse << pathName << "/linearETable_" << PCU_Comm_Self() << ".dat"; + ssq << pathName << "/linearQTable_" << m->getPCU()->Self() << ".dat"; + sse << pathName << "/linearETable_" << m->getPCU()->Self() << ".dat"; ssm << pathName << "/mesh_quality_vis"; ssl << pathName << "/mesh_edge_length_vis"; @@ -291,7 +291,7 @@ void getStats( } std::stringstream sstq; - sstq << pathName << "/linearBLTriQTable_" << PCU_Comm_Self() << ".dat"; + sstq << pathName << "/linearBLTriQTable_" << m->getPCU()->Self() << ".dat"; writeTable(sstq.str().c_str(), tri_qtable); } #endif diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index 927ca1aca..dc490efce 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -88,7 +88,7 @@ int main(int argc, char** argv) ma::adapt(ma_in); m->verify(); apf::writeVtkFiles("after",m); - if (in.prePhastaBalanceMethod != "none" && PCU_Comm_Peers() > 1) + if (in.prePhastaBalanceMethod != "none" && m->getPCU()->Peers() > 1) ph::balance(in,m); /* output restart and geombc */ chef::preprocess(m,in); diff --git a/test/pumi.cc b/test/pumi.cc index bda9ed4a8..8bcc88912 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -170,10 +170,10 @@ int main(int argc, char** argv) pMeshIter it = m->begin(dim); while ((e = m->iterate(it))) { - pid=pumi_ment_getID(e)%PCU_Comm_Peers(); + pid=pumi_ment_getID(e)%m->getPCU()->Peers(); plan->send(e, pid); if (pid-1>=0) plan->send(e, pid-1); - if (pid+1send(e, pid+1); + if (pid+1getPCU()->Peers()) plan->send(e, pid+1); if (count==5) break; ++count; } @@ -789,7 +789,7 @@ void TEST_GHOSTING(pMesh m) pumi_ghost_create(m, ghosting_plan); int total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; - MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); if (!pumi_rank()) std::cout<<"\n[test_pumi] element-wise pumi_ghost_create: #ghost increase="<getPCU()->GetMPIComm()); if (!pumi_rank()) std::cout<<"\n[test_pumi] layer-wise pumi_ghost_createLayer (bd "<getPCU()->GetMPIComm()); if (!pumi_rank()) std::cout<<"\n[test_pumi] accumulative pumi_ghost_createLayer (bd "<count(m->getDimension()); - numElems = PCU_Add_Long(numElems); - avgQuality = PCU_Add_Double(avgQuality); + numElems = m->getPCU()->Add(numElems); + avgQuality = m->getPCU()->Add(avgQuality); avgQuality /= numElems; - numElemsBelowTol = PCU_Add_Long(numElemsBelowTol); - avgQualityBelowTol = PCU_Add_Double(avgQualityBelowTol); + numElemsBelowTol = m->getPCU()->Add(numElemsBelowTol); + avgQualityBelowTol = m->getPCU()->Add(avgQualityBelowTol); if (numElemsBelowTol > 0) avgQualityBelowTol /= numElemsBelowTol; - minQuality = PCU_Min_Double(minQuality); + minQuality = m->getPCU()->Min(minQuality); } static void processMesh(apf::Mesh2* m) diff --git a/test/rm_extrusion.cc b/test/rm_extrusion.cc index a21133b8b..1f98a3e01 100644 --- a/test/rm_extrusion.cc +++ b/test/rm_extrusion.cc @@ -128,7 +128,7 @@ int main(int argc, char** argv) if(!PCU_Comm_Self()) fprintf(stderr, "Removed surface extrusion constraints\n"); M_write(sim_mesh, smsNew_path, 0, progress); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if(!PCU_Comm_Self()) fprintf(stderr, "read the mesh, removed the face extrusion attributes, and wrote the mesh %f seconds\n", t1-t0); diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc index bfb399a38..1d5b9ae3f 100644 --- a/test/runSimxAnisoAdapt.cc +++ b/test/runSimxAnisoAdapt.cc @@ -436,21 +436,21 @@ pMSAdapt addSizesToSimxMesh( double runSimxAdapt(pMSAdapt adapter) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); MSA_adapt(adapter, 0); MSA_delete(adapter); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } double runSimxMeshImprover(pMesh mesh, double minQuality) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); pVolumeMeshImprover vmi = VolumeMeshImprover_new(mesh); VolumeMeshImprover_setShapeMetric(vmi, ShapeMetricType_VolLenRatio, minQuality); VolumeMeshImprover_execute(vmi, 0); VolumeMeshImprover_delete(vmi); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } diff --git a/test/serialize.cc b/test/serialize.cc index f233770b4..77f25b2e9 100644 --- a/test/serialize.cc +++ b/test/serialize.cc @@ -48,7 +48,7 @@ int main( int argc, char* argv[]) GroupCode code; code.mesh = apf::loadMdsMesh(argv[1], argv[2]); code.meshFile = argv[3]; - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); Parma_ShrinkPartition(code.mesh, atoi(argv[4]), code); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); diff --git a/test/sim_part.cc b/test/sim_part.cc index fbf9e0f14..13d4427f0 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -68,7 +68,7 @@ int main(int argc, char **argv) MPI_Init(&argc,&argv); PCU_Comm_Init(); lion_set_verbosity(1); - PCU_Protect(); + pcu::Protect(); // Initialize PartitionedMesh - this should be the first Simmetrix call // Also initializes MPI in parallel SimPartitionedMesh_start(&argc, &argv); @@ -153,7 +153,7 @@ int main(int argc, char **argv) NM_release(nmodel); #endif - if (PCU_Comm_Self()==0) { + if (pmesh->getPCU()->Self()==0) { cout<<"**********************************"<end(it); - PCU_Max_Doubles(&maxSize, 1); + m->getPCU()->Max(&maxSize, 1); return maxSize; } diff --git a/test/vtxEdgeElmBalance.cc b/test/vtxEdgeElmBalance.cc index 7ce4b9ecc..39cce7e03 100644 --- a/test/vtxEdgeElmBalance.cc +++ b/test/vtxEdgeElmBalance.cc @@ -25,7 +25,7 @@ namespace { apf::MeshTag* setWeights(apf::Mesh* m, double edgeWeight) { double weights[4] = {1.,edgeWeight,1.,1.}; - weights[1] = (!PCU_Comm_Self()) ? edgeWeight : 1.0; + weights[1] = (!m->getPCU()->Self()) ? edgeWeight : 1.0; const int d = m->getDimension(); apf::MeshTag* tag = m->createDoubleTag("parma_weight", 1); setWeight(m, tag, 0, weights[0]); @@ -69,7 +69,7 @@ int main(int argc, char** argv) Parma_PrintWeightedPtnStats(m, weights, "initial"); const double step = 0.5; const int verbose = 1; apf::Balancer* balancer = Parma_MakeVtxEdgeElmBalancer(m, step, verbose); - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) fprintf(stderr, "STATUS target imbalance %.2f\n", targetImb); balancer->balance(weights, targetImb); delete balancer; diff --git a/test/writePart.cc b/test/writePart.cc index ee22d41fb..0f44254f2 100644 --- a/test/writePart.cc +++ b/test/writePart.cc @@ -19,7 +19,7 @@ int main(int argc, char** argv) } gmi_register_mesh(); apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); - if (PCU_Comm_Self() == atoi(argv[3])) + if (m->getPCU()->Self() == atoi(argv[3])) apf::writeMdsPart(m, argv[4]); m->destroyNative(); apf::destroyMesh(m); diff --git a/zoltan/apfInterElement.cc b/zoltan/apfInterElement.cc index c8fe29252..f69ed6f6f 100644 --- a/zoltan/apfInterElement.cc +++ b/zoltan/apfInterElement.cc @@ -41,16 +41,16 @@ static void packOtherGid(GlobalNumbering* gn, MeshEntity* s) Mesh* m = getMesh(gn); Copy other = getOtherSide(m, s); long gid = getElementGid(gn, getSideElement(m, s)); - PCU_COMM_PACK(other.peer, other.entity); - PCU_COMM_PACK(other.peer, gid); + m->getPCU()->Pack(other.peer, other.entity); + m->getPCU()->Pack(other.peer, gid); } static void unpackOtherGid(Mesh* m, MeshTag* t) { MeshEntity* s; - PCU_COMM_UNPACK(s); + m->getPCU()->Unpack(s); long gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); m->setLongTag(s, t, &gid); } @@ -59,14 +59,14 @@ MeshTag* tagOpposites(GlobalNumbering* gn, const char* name) Mesh* m = getMesh(gn); MeshTag* t = m->createLongTag(name, 1); int sideDim = m->getDimension() - 1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(sideDim); MeshEntity* e; while ((e = m->iterate(it))) if (hasOtherSide(m, e)) packOtherGid(gn, e); m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); while (PCU_Comm_Receive()) unpackOtherGid(m, t); return t; diff --git a/zoltan/apfZoltan.cc b/zoltan/apfZoltan.cc index bb04828bb..8039523d6 100644 --- a/zoltan/apfZoltan.cc +++ b/zoltan/apfZoltan.cc @@ -25,7 +25,7 @@ class ZoltanSplitter : public Splitter virtual ~ZoltanSplitter() {} virtual Migration* split(MeshTag* weights, double tolerance, int multiple) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Migration* plan = bridge.run(weights, tolerance, multiple); if (isSynchronous) { for (int i = 0; i < plan->count(); ++i) { @@ -35,8 +35,8 @@ class ZoltanSplitter : public Splitter plan->send(e, p); } } - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1, "planned Zoltan split factor %d to target" " imbalance %f in %f seconds\n", multiple, tolerance, t1 - t0); return plan; @@ -55,15 +55,15 @@ class ZoltanBalancer : public Balancer virtual ~ZoltanBalancer() {} virtual void balance(MeshTag* weights, double tolerance) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Migration* plan = bridge.run(weights, tolerance, 1); - if (!PCU_Comm_Self()) + if (!bridge.mesh->getPCU()->Self()) lion_oprint(1, "planned Zoltan balance to target " "imbalance %f in %f seconds\n", - tolerance, PCU_Time() - t0); + tolerance, pcu::Time() - t0); bridge.mesh->migrate(plan); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!bridge.mesh->getPCU()->Self()) lion_oprint(1,"Zoltan balanced to %f in %f seconds\n", tolerance, t1-t0); } diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index 1a0488df7..6fc125436 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -273,7 +273,7 @@ ZoltanData::ZoltanData(ZoltanMesh* zb_) : zb(zb_) if (zb->isLocal) comm = MPI_COMM_SELF; else - comm = PCU_Get_Comm(); + comm = zb->mesh->getPCU()->GetMPIComm(); ztn =Zoltan_Create(comm); import_gids = NULL; import_lids = NULL; @@ -325,7 +325,7 @@ void ZoltanData::setup() //Debug sprintf(paramStr, "%d", dbgLvl); - if ( zb->isLocal && 0 != PCU_Comm_Self() ) + if ( zb->isLocal && 0 != zb->mesh->getPCU()->Self() ) sprintf(paramStr, "%d", 0); //if local silence all but rank 0 Zoltan_Set_Param(ztn, "debug_level", paramStr); Zoltan_Set_Param(ztn, "PARMETIS_OUTPUT_LEVEL", paramStr); @@ -372,7 +372,7 @@ void ZoltanData::ptn() &import_to_part, &num_exported, &export_gids, &export_lids, &export_procs, &export_to_part); if( ZOLTAN_OK != ret ) { - if( 0 == PCU_Comm_Self() ) + if( 0 == zb->mesh->getPCU()->Self() ) lion_eprint(1, "ERROR Zoltan partitioning failed\n"); exit(EXIT_FAILURE); } From 2ec87e35445a06aa2893e2236ec8286384183c94 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 30 Jan 2024 15:03:46 -0500 Subject: [PATCH 042/141] Re-checked apf and pcu2 fix --- apf/apfCGNS.cc | 4 +- apf/apfCavityOp.cc | 2 +- apf/apfConstruct.cc | 66 +++++----- apf/apfConvert.cc | 32 ++--- apf/apfFieldData.cc | 20 +-- apf/apfMesh.cc | 16 +-- apf/apfMesh2.cc | 30 ++--- apf/apfMigrate.cc | 64 +++++----- apf/apfNumbering.cc | 2 +- apf/apfVerify.cc | 30 ++--- apf/apfVtk.cc | 18 +-- apf/apfVtkPieceWiseFields.cc | 6 +- pcu/CMakeLists.txt | 2 + pcu/PCU2.h | 24 ++-- pcu/pcu2.cc | 238 +++++++++++++++++------------------ 15 files changed, 279 insertions(+), 275 deletions(-) diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 6225b1c0d..69aab3246 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -548,7 +548,7 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); cgsize_t num = 0; for (int i = 0; i < m->getPCU()->Self(); i++) @@ -1031,7 +1031,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap // PCU_Barrier(); // } - PCU_Barrier(); + m->getPCU()->Barrier(); if (myRank == 0) { std::cout << "*******Global Mesh Stats*****************\n"; diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index a1600b2c2..c13c0c1d8 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -134,7 +134,7 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = PCU_Min_Int(requests.empty()); + int done = mesh->getPCU()->Min((int)requests.empty()); if (done) return false; /* throw in the local pull requests */ int self = mesh->getPCU()->Self(); diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index e9bd33e7d..e461ecca1 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -102,7 +102,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) } Gid tmpL=gid / quotient; int tmpI=tmpL; int to = std::min(peers - 1,tmpI); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } m->getPCU()->Send(); Gid myOffset = (long)self * quotient; @@ -126,10 +126,10 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) int to = parts[j]; Gid gid = i + myOffset; int nparts = parts.size(); - PCU_COMM_PACK(to, gid); - PCU_COMM_PACK(to, nparts); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, nparts); for (size_t k = 0; k < parts.size(); ++k) - PCU_COMM_PACK(to, parts[k]); + m->getPCU()->Pack(to, parts[k]); } } m->getPCU()->Send(); @@ -168,8 +168,8 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert) m->getResidence(vert, residence); APF_ITERATE(Parts, residence, rit) if (*rit != self) { - PCU_COMM_PACK(*rit, gid); - PCU_COMM_PACK(*rit, vert); + m->getPCU()->Pack(*rit, gid); + m->getPCU()->Pack(*rit, vert); } } m->getPCU()->Send(); @@ -228,7 +228,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, This means we might need to send and recv some coords */ double* c = new double[mySize*3]; - Gid start = PCU_Exscan_Long(nverts); + Gid start = m->getPCU()->Exscan(nverts); m->getPCU()->Begin(); // the forced 64 bit math below may not be necessary Gid tmpL=start / quotient; @@ -239,9 +239,9 @@ void setCoords(Mesh2* m, const double* coords, int nverts, int n = std::min(tmpInt, nverts); while (nverts > 0) { - PCU_COMM_PACK(to, start); - PCU_COMM_PACK(to, n); - PCU_Comm_Pack(to, coords, n*3*sizeof(double)); + m->getPCU()->Pack(to, start); + m->getPCU()->Pack(to, n); + m->getPCU()->Pack(to, coords, n*3*sizeof(double)); nverts -= n; start += n; @@ -249,11 +249,11 @@ void setCoords(Mesh2* m, const double* coords, int nverts, to = std::min(peers - 1, to + 1); n = std::min(quotient, nverts); } - PCU_Comm_Send(); + m->getPCU()->Send(); while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(start); PCU_COMM_UNPACK(n); // |||||| more in-place 64 bit math - PCU_Comm_Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double)); + m->getPCU()->Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double)); } /* Tell all the owners of the coords what we need */ @@ -265,7 +265,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, Gid tmpL=gid / quotient; int tmpInt=tmpL; int to = std::min(peers - 1, tmpInt); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } m->getPCU()->Send(); while (m->getPCU()->Receive()) { @@ -284,8 +284,8 @@ void setCoords(Mesh2* m, const double* coords, int nverts, for (size_t j = 0; j < parts.size(); ++j) { int to = parts[j]; Gid gid = i + myOffset; - PCU_COMM_PACK(to, gid); - PCU_Comm_Pack(to, &c[i*3], 3*sizeof(double)); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, &c[i*3], 3*sizeof(double)); } } m->getPCU()->Send(); @@ -293,7 +293,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, Gid gid; PCU_COMM_UNPACK(gid); double v[3]; - PCU_Comm_Unpack(v, sizeof(v)); + m->getPCU()->Unpack(v, sizeof(v)); Vector3 vv(v); m->setPoint(globalToVert[gid], 0, vv); } @@ -318,7 +318,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, /* Force each peer to have exactly mySize verts. This means we might need to send and recv some matches */ Gid* c = new Gid[mySize]; - Gid start = PCU_Exscan_Long(nverts); + Gid start = m->getPCU()->Exscan(nverts); m->getPCU()->Begin(); @@ -329,9 +329,9 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, tmpInt=tmpL; int n = std::min(tmpInt, nverts); while (nverts > 0) { - PCU_COMM_PACK(to, start); - PCU_COMM_PACK(to, n); - PCU_Comm_Pack(to, matches, n*sizeof(Gid)); + m->getPCU()->Pack(to, start); + m->getPCU()->Pack(to, n); + m->getPCU()->Pack(to, matches, n*sizeof(Gid)); nverts -= n; start += n; @@ -343,7 +343,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, while (m->getPCU()->Receive()) { PCU_COMM_UNPACK(start); PCU_COMM_UNPACK(n); /// in-place 64 - PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(Gid)); + m->getPCU()->Unpack(&c[(start - myOffset)], n*sizeof(Gid)); } @@ -355,7 +355,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, Gid gid = it->first; int tmpI=gid / quotient; int to = std::min(peers - 1,tmpI); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } m->getPCU()->Send(); while (m->getPCU()->Receive()) { @@ -374,8 +374,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, int to = parts[j]; Gid gid = i + myOffset; Gid matchGid = c[i]; - PCU_COMM_PACK(to, gid); - PCU_COMM_PACK(to, matchGid); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, matchGid); } } m->getPCU()->Send(); @@ -400,8 +400,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, Gid gid = it->first; int tmpI=gid / quotient; int to = std::min(peers - 1,tmpI); - PCU_COMM_PACK(to, gid); - PCU_COMM_PACK(to, e); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, e); } m->getPCU()->Send(); typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs; @@ -428,8 +428,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, if( matchGid != -1 ) { // marker for an unmatched vertex int tmpI=matchGid / quotient; int to = std::min(peers - 1,tmpI); //broker - PCU_COMM_PACK(to, gid); // send the local vert gid - PCU_COMM_PACK(to, matchGid); // and the match gid needed + m->getPCU()->Pack(to, gid); // send the local vert gid + m->getPCU()->Pack(to, matchGid); // and the match gid needed } } m->getPCU()->Send(); @@ -453,16 +453,16 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, std::vector parts = it->second; for(size_t i=0; igetPCU()->Pack(to, gid); + m->getPCU()->Pack(to, matchGid); size_t numMatches = gidPtrs[matchGid-myOffset].size(); - PCU_COMM_PACK(to, numMatches); + m->getPCU()->Pack(to, numMatches); for( size_t i=0; igetPCU()->Pack(to, owner); apf::MeshEntity* ent = eop.second; - PCU_COMM_PACK(to, ent); + m->getPCU()->Pack(to, ent); } } } diff --git a/apf/apfConvert.cc b/apf/apfConvert.cc index 70ec499f9..9b5227973 100644 --- a/apf/apfConvert.cc +++ b/apf/apfConvert.cc @@ -161,7 +161,7 @@ class Converter /* creating links backwards is OK because they go both ways; as long as every link gets a backward copy they will all get copies */ - PCU_Comm_Begin(); + inMesh->getPCU()->Begin(); MeshIterator* it = inMesh->begin(dim); MeshEntity *oldLeft; while ((oldLeft = inMesh->iterate(it))) @@ -173,18 +173,18 @@ class Converter { int rightPart = it->first; MeshEntity* oldRight = it->second; - PCU_COMM_PACK(rightPart,oldRight); - PCU_COMM_PACK(rightPart,newLeft); + inMesh->getPCU()->Pack(rightPart,oldRight); + inMesh->getPCU()->Pack(rightPart,newLeft); } } inMesh->end(it); - PCU_Comm_Send(); + inMesh->getPCU()->Send(); /* accumulation of residence sets... we could also constantly reclassify... */ std::map > map_ent; - while (PCU_Comm_Listen()) + while (inMesh->getPCU()->Listen()) { - int leftPart = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int leftPart = inMesh->getPCU()->Sender(); + while ( ! inMesh->getPCU()->Unpacked()) { MeshEntity* oldRight; PCU_COMM_UNPACK(oldRight); @@ -205,7 +205,7 @@ class Converter Copies remotes; Parts parts; parts.insert(vecEnt.begin(), vecEnt.end()); - parts.insert(PCU_Comm_Self()); + parts.insert(inMesh->getPCU()->Self()); outMesh->setResidence(newE, parts); } } @@ -407,7 +407,7 @@ class Converter { if (inMesh->getShape() != getLagrange(2) && inMesh->getShape() != getSerendipity()) return; - if ( ! PCU_Comm_Self()) + if ( ! inMesh->getPCU()->Self()) lion_eprint(1,"transferring quadratic mesh\n"); changeMeshShape(outMesh,inMesh->getShape(),/*project=*/false); convertField(inMesh->getCoordinateField(),outMesh->getCoordinateField()); @@ -415,7 +415,7 @@ class Converter void createMatches(int dim) { /* see createRemotes for the algorithm comments */ - PCU_Comm_Begin(); + inMesh->getPCU()->Begin(); MeshIterator* it = inMesh->begin(dim); MeshEntity *oldLeft; while ((oldLeft = inMesh->iterate(it))) @@ -427,16 +427,16 @@ class Converter { int rightPart = matches[i].peer; MeshEntity* oldRight = matches[i].entity; - PCU_COMM_PACK(rightPart,oldRight); - PCU_COMM_PACK(rightPart,newLeft); + inMesh->getPCU()->Pack(rightPart,oldRight); + inMesh->getPCU()->Pack(rightPart,newLeft); } } inMesh->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + inMesh->getPCU()->Send(); + while (inMesh->getPCU()->Listen()) { - int leftPart = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int leftPart = inMesh->getPCU()->Sender(); + while ( ! inMesh->getPCU()->Unpacked()) { MeshEntity* oldRight; PCU_COMM_UNPACK(oldRight); diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc index 2d1541b71..a57629ead 100644 --- a/apf/apfFieldData.cc +++ b/apf/apfFieldData.cc @@ -46,15 +46,15 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) shr->getCopies(e, copies); for (size_t i = 0; i < copies.getSize(); ++i) { - PCU_COMM_PACK(copies[i].peer, copies[i].entity); - PCU_Comm_Pack(copies[i].peer, &(values[0]), n*sizeof(T)); + m->getPCU()->Pack(copies[i].peer, copies[i].entity); + m->getPCU()->Pack(copies[i].peer, &(values[0]), n*sizeof(T)); } apf::Copies ghosts; if (m->getGhosts(e, ghosts)) APF_ITERATE(Copies, ghosts, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_Comm_Pack(it->first, &(values[0]), n*sizeof(T)); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, &(values[0]), n*sizeof(T)); } } m->end(it); @@ -65,7 +65,7 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) PCU_COMM_UNPACK(e); int n = f->countValuesOn(e); NewArray values(n); - PCU_Comm_Unpack(&(values[0]),n*sizeof(T)); + m->getPCU()->Unpack(&(values[0]),n*sizeof(T)); data->set(e,&(values[0])); } } @@ -122,8 +122,8 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c for (size_t i = 0; i < copies.getSize(); ++i) { - PCU_COMM_PACK(copies[i].peer, copies[i].entity); - PCU_Comm_Pack(copies[i].peer, &(values[0]), n*sizeof(double)); + m->getPCU()->Pack(copies[i].peer, copies[i].entity); + m->getPCU()->Pack(copies[i].peer, &(values[0]), n*sizeof(double)); } // ghosts - only do them if this entity is on a partition boundary @@ -133,8 +133,8 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c if (m->getGhosts(e, ghosts)) APF_ITERATE(Copies, ghosts, it2) { - PCU_COMM_PACK(it2->first, it2->second); - PCU_Comm_Pack(it2->first, &(values[0]), n*sizeof(double)); + m->getPCU()->Pack(it2->first, it2->second); + m->getPCU()->Pack(it2->first, &(values[0]), n*sizeof(double)); } } } @@ -150,7 +150,7 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c int n = f->countValuesOn(e); NewArray values(n); NewArray inValues(n); - PCU_Comm_Unpack(&(inValues[0]),n*sizeof(double)); + m->getPCU()->Unpack(&(inValues[0]),n*sizeof(double)); data->get(e,&(values[0])); for (int i = 0; i < n; ++i) values[i] = reduce_op.apply(values[i], inValues[i]); diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 41ea506e2..3e140864b 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -868,8 +868,8 @@ void printTypes(Mesh* m) while ((e = m->iterate(it))) typeCnt[m->getType(e)]++; m->end(it); - PCU_Add_Longs(typeCnt,Mesh::TYPES); - if (!PCU_Comm_Self()) { + m->getPCU()->Add(typeCnt,Mesh::TYPES); + if (!m->getPCU()->Self()) { lion_oprint(1,"number of"); for (int i=0; igetPCU()->Add(n, 4); printTypes(m); if (!PCU_Comm_Self()) lion_oprint(1,"mesh entity counts: v %ld e %ld f %ld r %ld\n", @@ -895,8 +895,8 @@ void warnAboutEmptyParts(Mesh* m) int emptyParts = 0; if (!m->count(m->getDimension())) ++emptyParts; - emptyParts = PCU_Add_Int(emptyParts); - if (emptyParts && (!PCU_Comm_Self())) + emptyParts = m->getPCU()->Add(emptyParts); + if (emptyParts && (!m->getPCU()->Self())) lion_eprint(1,"APF warning: %d empty parts\n",emptyParts); } @@ -1023,7 +1023,7 @@ void MatchedSharing::formCountMap() Parts neighbors; getNeighbors(neighbors); APF_ITERATE(Parts, neighbors, nit) - PCU_COMM_PACK(*nit, count); + mesh->getPCU()->Pack(*nit, count); mesh->getPCU()->Send(); while (mesh->getPCU()->Receive()) { size_t oc; @@ -1194,10 +1194,10 @@ void packTagInfo(Mesh* m, MeshTag* t, int to) packString(name, to); int type; type = m->getTagType(t); - PCU_COMM_PACK(to, type); + m->getPCU()->Pack(to, type); int size; size = m->getTagSize(t); - PCU_COMM_PACK(to, size); + m->getPCU()->Pack(to, size); } void unpackTagInfo(std::string& name, int& type, int& size) diff --git a/apf/apfMesh2.cc b/apf/apfMesh2.cc index 0f070e679..cfcd687b5 100644 --- a/apf/apfMesh2.cc +++ b/apf/apfMesh2.cc @@ -149,17 +149,17 @@ static void getCandidateParts(Mesh* m, MeshEntity* e, Parts& parts) static void packProposal(Mesh* m, MeshEntity* e, int to) { int t = m->getType(e); - PCU_COMM_PACK(to,t); - PCU_COMM_PACK(to,e); + m->getPCU()->Pack(to,t); + m->getPCU()->Pack(to,e); int d = getDimension(m, e); Downward down; int nd = m->getDownward(e, d - 1, down); - PCU_COMM_PACK(to,nd); + m->getPCU()->Pack(to,nd); for (int i = 0; i < nd; ++i) { Copies remotes; m->getRemotes(down[i], remotes); MeshEntity* dr = remotes[to]; - PCU_COMM_PACK(to,dr); + m->getPCU()->Pack(to,dr); } } @@ -179,7 +179,7 @@ void stitchMesh(Mesh2* m) int d_max = m->getDimension(); MeshEntity* e; for (int d=1; d < d_max; ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); while ((e = m->iterate(it))) { Parts candidateParts; @@ -189,10 +189,10 @@ void stitchMesh(Mesh2* m) packProposal(m, e,*pit); } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int from = PCU_Comm_Sender(); - while (!PCU_Comm_Unpacked()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { + int from = m->getPCU()->Sender(); + while (!m->getPCU()->Unpacked()) { int t; Downward da; unpackProposal(t, e, da); @@ -212,10 +212,10 @@ static void packTagClone(Mesh2* m, MeshTag* t, int to) packString(name, to); int type; type = m->getTagType(t); - PCU_COMM_PACK(to, type); + m->getPCU()->Pack(to, type); int size; size = m->getTagSize(t); - PCU_COMM_PACK(to, size); + m->getPCU()->Pack(to, size); } static MeshTag* unpackTagClone(Mesh2* m) @@ -239,7 +239,7 @@ static void packTagClones(Mesh2* m, int to) DynamicArray tags; m->getTags(tags); int n = tags.getSize(); - PCU_COMM_PACK(to, n); + m->getPCU()->Pack(to, n); /* warning! this loop goes backward to cater to MDS implementation-specific behavior. please forgive me. */ @@ -260,9 +260,9 @@ static void packFieldClone(Field* f, int to) std::string name = f->getName(); packString(name, to); int valueType = f->getValueType(); - PCU_COMM_PACK(to, valueType); + f->getMesh()->getPCU()->Pack(to, valueType); int components = f->countComponents(); - PCU_COMM_PACK(to, components); + f->getMesh()->getPCU()->Pack(to, components); std::string shapeName = f->getShape()->getName(); packString(shapeName, to); /* warning! this only supports tag-stored fields */ @@ -285,7 +285,7 @@ static Field* unpackFieldClone(Mesh2* m) static void packFieldClones(Mesh2* m, int to) { int n = m->countFields(); - PCU_COMM_PACK(to, n); + m->getPCU()->Pack(to, n); for (int i = 0; i < n; ++i) packFieldClone(m->getField(i), to); } diff --git a/apf/apfMigrate.cc b/apf/apfMigrate.cc index caeb1e8f6..55ad9d8c8 100644 --- a/apf/apfMigrate.cc +++ b/apf/apfMigrate.cc @@ -59,13 +59,13 @@ static void getAffected( Copies remotes; m->getRemotes(adjacent[i],remotes); APF_ITERATE(Copies,remotes,rit) - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); if (m->hasMatching()) { Matches matches; m->getMatches(adjacent[i],matches); for (size_t j=0; j < matches.getSize(); ++j) - PCU_COMM_PACK(matches[j].peer,matches[j].entity); + m->getPCU()->Pack(matches[j].peer,matches[j].entity); } }//downward adjacent loop }//upward affected loop @@ -128,8 +128,8 @@ void reduceMatchingToSenders( for (size_t j=0; j < matches.getSize(); ++j) { /* advertise to the match that this is a sender */ int to = matches[j].peer; - PCU_COMM_PACK(to,matches[j].entity); - PCU_COMM_PACK(to,e); + m->getPCU()->Pack(to,matches[j].entity); + m->getPCU()->Pack(to,e); } /* now forget all other matchings */ m->clearMatches(e); @@ -222,7 +222,7 @@ static void updateResidences( m->getRemotes(entity,remotes); APF_ITERATE(Copies,remotes,rit) { - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); packParts(rit->first,newResidence); } } @@ -262,12 +262,12 @@ static void packCommon( int to, MeshEntity* e) { - PCU_COMM_PACK(to,e); + m->getPCU()->Pack(to,e); ModelEntity* me = m->toModel(e); int modelType = m->getModelType(me); - PCU_COMM_PACK(to,modelType); + m->getPCU()->Pack(to,modelType); int modelTag = m->getModelTag(me); - PCU_COMM_PACK(to,modelTag); + m->getPCU()->Pack(to,modelTag); Parts residence; m->getResidence(e,residence); packParts(to,residence); @@ -294,9 +294,9 @@ static void packVertex( { Vector3 p; m->getPoint(e,0,p); - PCU_COMM_PACK(to,p); + m->getPCU()->Pack(to,p); m->getParam(e,p); - PCU_COMM_PACK(to,p); + m->getPCU()->Pack(to,p); } MeshEntity* unpackVertex( @@ -321,7 +321,7 @@ static void packReference( if (found!=remotes.end()) { MeshEntity* remote = found->second; - PCU_COMM_PACK(to,remote); + m->getPCU()->Pack(to,remote); } else { @@ -330,7 +330,7 @@ static void packReference( found = ghosts.find(to); PCU_ALWAYS_ASSERT(found!=ghosts.end()); MeshEntity* ghost = found->second; - PCU_COMM_PACK(to,ghost); + m->getPCU()->Pack(to,ghost); } } @@ -339,7 +339,7 @@ static void packDownward(Mesh2* m, int to, MeshEntity* e) Downward down; int d = getDimension(m, e); int n = m->getDownward(e,d-1,down); - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); for (int i=0; i < n; ++i) packReference(m,to,down[i]); } @@ -381,26 +381,26 @@ static void packTags( for (size_t i=0; i < total; ++i) if (m->hasTag(e,tags[i])) ++n; - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); for (size_t i=0; i < total; ++i) { MeshTag* tag = tags[i]; if (m->hasTag(e,tag)) { - PCU_COMM_PACK(to,i); + m->getPCU()->Pack(to,i); int type = m->getTagType(tag); int size = m->getTagSize(tag); if (type == Mesh2::DOUBLE) { DynamicArray d(size); m->getDoubleTag(e,tag,&(d[0])); - PCU_Comm_Pack(to,&(d[0]),size*sizeof(double)); + m->getPCU()->Pack(to,&(d[0]),size*sizeof(double)); } if (type == Mesh2::INT) { DynamicArray d(size); m->getIntTag(e,tag,&(d[0])); - PCU_Comm_Pack(to,&(d[0]),size*sizeof(int)); + m->getPCU()->Pack(to,&(d[0]),size*sizeof(int)); } } } @@ -415,13 +415,13 @@ void packRemotes( Copies remotes; m->getRemotes(e,remotes); size_t n = remotes.size(); - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); APF_ITERATE(Copies,remotes,rit) { int p=rit->first; MeshEntity* remote = rit->second; - PCU_COMM_PACK(to,p); - PCU_COMM_PACK(to,remote); + m->getPCU()->Pack(to,p); + m->getPCU()->Pack(to,remote); } } @@ -458,13 +458,13 @@ void unpackTags( if (type == Mesh2::DOUBLE) { DynamicArray d(size); - PCU_Comm_Unpack(&(d[0]),size*sizeof(double)); + m->getPCU()->Unpack(&(d[0]),size*sizeof(double)); m->setDoubleTag(e,tag,&(d[0])); } if (type == Mesh2::INT) { DynamicArray d(size); - PCU_Comm_Unpack(&(d[0]),size*sizeof(int)); + m->getPCU()->Unpack(&(d[0]),size*sizeof(int)); m->setIntTag(e,tag,&(d[0])); } } @@ -478,7 +478,7 @@ void packEntity( bool ghosting) { int type = m->getType(e); - PCU_COMM_PACK(to,type); + m->getPCU()->Pack(to,type); packCommon(m,to,e); if (type == Mesh::VERTEX) packVertex(m,to,e); @@ -557,8 +557,8 @@ static void echoRemotes( m->getRemotes(entity,temp); int from = temp.begin()->first; MeshEntity* sender = temp.begin()->second; - PCU_COMM_PACK(from,sender); - PCU_COMM_PACK(from,entity); + m->getPCU()->Pack(from,sender); + m->getPCU()->Pack(from,entity); } } @@ -645,7 +645,7 @@ static void bcastRemotes( getNewCopies(m,e,allRemotes,newCopies); APF_ITERATE(Copies,allRemotes,rit) { - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); packCopies(rit->first,newCopies); } newCopies.erase(rank); @@ -720,13 +720,13 @@ static void updateSenderMatching( m->getResidence(e,residence); /* pack the remote copies to itself, then add itself to the copies if it remains */ - PCU_COMM_PACK(self,e); + m->getPCU()->Pack(self,e); packCopies(self,copies); if (residence.count(self)) copies[self]=e; for (size_t j=0; j < matches.getSize(); ++j) { /* pack all copies to matched senders */ - PCU_COMM_PACK(matches[j].peer,matches[j].entity); + m->getPCU()->Pack(matches[j].peer,matches[j].entity); packCopies(matches[j].peer,copies); } } @@ -777,13 +777,13 @@ static void bcastMatching( APF_ITERATE(Copies,remotes,rit) { int to = rit->first; - PCU_COMM_PACK(to,rit->second); + m->getPCU()->Pack(to,rit->second); size_t n = matches.getSize(); - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); for (size_t j=0; j < n; ++j) { - PCU_COMM_PACK(to,matches[j].peer); - PCU_COMM_PACK(to,matches[j].entity); + m->getPCU()->Pack(to,matches[j].peer); + m->getPCU()->Pack(to,matches[j].entity); } } } diff --git a/apf/apfNumbering.cc b/apf/apfNumbering.cc index 3130aed9e..1ecb11af1 100644 --- a/apf/apfNumbering.cc +++ b/apf/apfNumbering.cc @@ -408,7 +408,7 @@ static void synchronizeEntitySet( Copies remotes; m->getRemotes(*it,remotes); APF_ITERATE(Copies,remotes,rit) - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); } m->getPCU()->Send(); while (m->getPCU()->Receive()) diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc index 5c2ad968b..0f48755e6 100644 --- a/apf/apfVerify.cc +++ b/apf/apfVerify.cc @@ -264,7 +264,7 @@ static void sendAllCopies(Mesh* m, MeshEntity* e) PCU_ALWAYS_ASSERT(!r.count(m->getPCU()->Self())); APF_ITERATE(Copies, r, it) { - PCU_COMM_PACK(it->first, it->second); + m->getPCU()->Pack(it->first, it->second); packCopies(it->first, a); } } @@ -304,8 +304,8 @@ static void sendGhostCopies(Mesh* m, MeshEntity* e) PCU_ALWAYS_ASSERT(g.size()); APF_ITERATE(Copies, g, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_COMM_PACK(it->first, e); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, e); } } @@ -419,9 +419,9 @@ static void sendCoords(Mesh* m, MeshEntity* e) m->getRemotes(e, r); APF_ITERATE(Copies, r, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_COMM_PACK(it->first, x); - PCU_COMM_PACK(it->first, p); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, x); + m->getPCU()->Pack(it->first, p); } } @@ -455,7 +455,7 @@ static long verifyCoords(Mesh* m) while (m->getPCU()->Receive()) if (!receiveCoords(m)) ++n; - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } long verifyVolumes(Mesh* m, bool printVolumes) @@ -481,12 +481,12 @@ long verifyVolumes(Mesh* m, bool printVolumes) } } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } static void packAlignment(Mesh* m, MeshEntity* e, MeshEntity* r, int to) { - PCU_COMM_PACK(to,r); + m->getPCU()->Pack(to,r); int d = getDimension(m, e); Downward down; int nd = m->getDownward(e, d - 1, down); @@ -494,7 +494,7 @@ static void packAlignment(Mesh* m, MeshEntity* e, MeshEntity* r, int to) Copies remotes; m->getRemotes(down[i], remotes); MeshEntity* dr = remotes[to]; - PCU_COMM_PACK(to,dr); + m->getPCU()->Pack(to,dr); } } @@ -565,7 +565,7 @@ static void verifyFields(Mesh* m) m->getPCU()->Begin(); if (self) { - PCU_COMM_PACK(self - 1, n); + m->getPCU()->Pack(self - 1, n); for (int i = 0; i < n; ++i) packFieldInfo(fields[i], self - 1); } @@ -721,7 +721,7 @@ static void receiveTagData(Mesh* m, DynamicArray& tags) } // switch } // while - int global_size = PCU_Max_Int((int)mismatch_tags.size()); + int global_size = m->getPCU()->Max((int)mismatch_tags.size()); if (global_size&&!m->getPCU()->Self()) for (std::set::iterator it=mismatch_tags.begin(); it!=mismatch_tags.end(); ++it) lion_oprint(1," - tag \"%s\" data mismatch over remote/ghost copies\n", m->getTagName(*it)); @@ -737,7 +737,7 @@ static void verifyTags(Mesh* m) m->getPCU()->Begin(); if (self) { - PCU_COMM_PACK(self - 1, n); + m->getPCU()->Pack(self - 1, n); for (int i = 0; i < n; ++i) packTagInfo(m, tags[i], self - 1); } @@ -799,7 +799,7 @@ static void verifyTags(Mesh* m) void verify(Mesh* m, bool abort_on_error) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); verifyTags(m); verifyFields(m); @@ -833,7 +833,7 @@ void verify(Mesh* m, bool abort_on_error) n = verifyVolumes(m); if (n && (!m->getPCU()->Self())) lion_eprint(1,"apf::verify warning: %ld negative simplex elements\n", n); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!m->getPCU()->Self()) lion_oprint(1,"mesh verified in %f seconds\n", t1 - t0); } diff --git a/apf/apfVtk.cc b/apf/apfVtk.cc index 98f09985b..8915dd55d 100644 --- a/apf/apfVtk.cc +++ b/apf/apfVtk.cc @@ -832,9 +832,9 @@ static void writeVtuFile(const char* prefix, bool isWritingBinary, int cellDim) { - double t0 = PCU_Time(); - std::string fileName = getPieceFileName(PCU_Comm_Self()); - std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, PCU_Comm_Self()); + double t0 = pcu::Time(); + std::string fileName = getPieceFileName(n->getMesh()->getPCU()->Self()); + std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, n->getMesh()->getPCU()->Self()); std::stringstream buf; Mesh* m = n->getMesh(); DynamicArray nodes; @@ -874,8 +874,8 @@ static void writeVtuFile(const char* prefix, buf << "\n"; buf << "\n"; buf << "\n"; - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile into buffers: %f seconds\n", t1 - t0); } @@ -884,8 +884,8 @@ static void writeVtuFile(const char* prefix, PCU_ALWAYS_ASSERT(file.is_open()); file << buf.rdbuf(); } - double t2 = PCU_Time(); - if (!PCU_Comm_Self()) + double t2 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile buffers to disk: %f seconds\n", t2 - t1); } @@ -928,7 +928,7 @@ void writeVtkFilesRunner(const char* prefix, int cellDim) { if (cellDim == -1) cellDim = m->getDimension(); - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (!m->getPCU()->Self()) { safe_mkdir(prefix); @@ -939,7 +939,7 @@ void writeVtkFilesRunner(const char* prefix, Numbering* n = numberOverlapNodes(m,"apf_vtk_number"); m->removeNumbering(n); writeVtuFile(prefix, n, writeFields, isWritingBinary, cellDim); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!m->getPCU()->Self()) { lion_oprint(1,"vtk files %s written in %f seconds\n", prefix, t1 - t0); diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc index 4879d4cc0..f57527b0a 100644 --- a/apf/apfVtkPieceWiseFields.cc +++ b/apf/apfVtkPieceWiseFields.cc @@ -72,7 +72,7 @@ static std::string getFileNameAndPathVtu(const char* prefix, static void writeNedelecVtkFile(const char* prefix, Mesh* m, std::vector writeFields) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); // get the number of points on this part MeshEntity* e; @@ -190,7 +190,7 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, } m->end(it); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile into buffers: %f seconds\n", t1 - t0); @@ -200,7 +200,7 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, PCU_ALWAYS_ASSERT(file.is_open()); file << buf.rdbuf(); } - double t2 = PCU_Time(); + double t2 = pcu::Time(); if (!m->getPCU()->Self()) { lion_oprint(1,"writeNedelecVtkFile buffers to disk: %f seconds\n", t2 - t1); diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index bd464ecaf..ec7c3c266 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -10,6 +10,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES pcu.cc + pcu2.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -27,6 +28,7 @@ set(SOURCES # Package headers set(HEADERS PCU.h + PCU2.h pcu_io.h pcu_util.h reel/reel.h diff --git a/pcu/PCU2.h b/pcu/PCU2.h index ac62ec10b..2d82507ef 100644 --- a/pcu/PCU2.h +++ b/pcu/PCU2.h @@ -13,8 +13,12 @@ extern "C" { #include #endif -int PCU_Comm_Init2(PCUHandle& h); -int PCU_Comm_Free2(PCUHandle& h); +struct PCUHandle { + void* ptr; +}; + +int PCU_Comm_Init2(PCUHandle* h); +int PCU_Comm_Free2(PCUHandle* h); /*rank/size functions*/ int PCU_Comm_Self2(PCUHandle h); @@ -23,16 +27,16 @@ int PCU_Comm_Peers2(PCUHandle h); /*recommended message passing API*/ void PCU_Comm_Begin2(PCUHandle h); int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void* data, size_t size); -#define PCU_COMM_PACK2(to_rank,object)\ -PCU_Comm_Pack2(to_rank,&(object),sizeof(object)) +#define PCU_COMM_PACK2(handle, to_rank,object)\ +PCU_Comm_Pack2(handle, to_rank,&(object),sizeof(object)) int PCU_Comm_Send2(PCUHandle h); bool PCU_Comm_Receive2(PCUHandle h); bool PCU_Comm_Listen2(PCUHandle h); int PCU_Comm_Sender2(PCUHandle h); bool PCU_Comm_Unpacked2(PCUHandle h); int PCU_Comm_Unpack2(PCUHandle h, void* data, size_t size); -#define PCU_COMM_UNPACK(object)\ -PCU_Comm_Unpack2(&(object),sizeof(object)) +#define PCU_COMM_UNPACK2(handle, object)\ +PCU_Comm_Unpack2(handle, &(object),sizeof(object)) /*turns deterministic ordering for the above API on/off*/ @@ -75,14 +79,14 @@ int PCU_Proc_Peers2(PCUHandle h); /*IPComMan replacement API*/ int PCU_Comm_Write2(PCUHandle h, int to_rank, const void* data, size_t size); -#define PCU_COMM_WRITE(to,data) \ -PCU_Comm_Write2(to,&(data),sizeof(data)) +#define PCU_COMM_WRITE2(handle,to,data) \ +PCU_Comm_Write2(handle, to,&(data),sizeof(data)) bool PCU_Comm_Read2(PCUHandle h, int* from_rank, void** data, size_t* size); /*Debug file I/O API*/ -void PCU_Debug_Open(PCUHandle h); +void PCU_Debug_Open2(PCUHandle h); -void PCU_Debug_Print2(PCUHandle h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(1,2); +void PCU_Debug_Print2(PCUHandle h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(2,3); /*lesser-used APIs*/ bool PCU_Comm_Initialized2(PCUHandle h); int PCU_Comm_Packed2(PCUHandle h ,int to_rank, size_t* size); diff --git a/pcu/pcu2.cc b/pcu/pcu2.cc index fdb79c7a9..73b414d42 100644 --- a/pcu/pcu2.cc +++ b/pcu/pcu2.cc @@ -3,13 +3,11 @@ #include "reel.h" #include -struct PCUHandle { - void* ptr -}; + extern "C" { -int PCU_Comm_Init2(PCUHandle& h) { +int PCU_Comm_Init2(PCUHandle* h) { if (h->ptr != nullptr) reel_fail("nested calls to Comm_Init"); pcu::PCU* pcu_object = new pcu::PCU(MPI_COMM_WORLD); @@ -18,30 +16,30 @@ int PCU_Comm_Init2(PCUHandle& h) { return PCU_SUCCESS; } -int PCU_Comm_Free2(PCUHandle& h) { +int PCU_Comm_Free2(PCUHandle* h) { if (h->ptr == nullptr) reel_fail("Comm_Free called before Comm_Init"); - delete h->ptr; + delete static_cast(h->ptr); h->ptr = nullptr; return PCU_SUCCESS; } int PCU_Comm_Self2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Self called before Comm_Init"); - return static_cast(h->ptr)->Self(); + return static_cast(h.ptr)->Self(); } int PCU_Comm_Peers2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Peers called before Comm_Init"); - return static_cast(h->ptr)->Peers(); + return static_cast(h.ptr)->Peers(); } void PCU_Comm_Begin2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Begin called before Comm_Init"); - static_cast(h->ptr)->Begin(); + static_cast(h.ptr)->Begin(); } /** \brief Packs data to be sent to \a to_rank. @@ -51,9 +49,9 @@ void PCU_Comm_Begin2(PCUHandle h) { PCU_Comm_Send. */ int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void *data, size_t size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Pack called before Comm_Init"); - return static_cast(h->ptr)->Pack(to_rank, data, size); + return static_cast(h.ptr)->Pack(to_rank, data, size); } /** \brief Sends all buffers for this communication phase. @@ -64,9 +62,9 @@ int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void *data, size_t size) { may begin after this call. */ int PCU_Comm_Send2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Send called before Comm_Init"); - return static_cast(h->ptr)->Send(); + return static_cast(h.ptr)->Send(); } /** \brief Tries to receive a buffer for this communication phase. @@ -81,24 +79,24 @@ int PCU_Comm_Send2(PCUHandle h) { again, because the previously received buffer is destroyed by the call. */ bool PCU_Comm_Listen2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Listen called before Comm_Init"); - return static_cast(h->ptr)->Listen(); + return static_cast(h.ptr)->Listen(); } /** \brief Returns in * \a from_rank the sender of the current received buffer. \details This function should be called after a successful PCU_Comm_Listen. */ int PCU_Comm_Sender2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Sender called before Comm_Init"); - return static_cast(h->ptr)->Sender(); + return static_cast(h.ptr)->Sender(); } bool PCU_Comm_Unpacked2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Unpacked called before Comm_Init"); - return static_cast(h->ptr)->Unpacked(); + return static_cast(h.ptr)->Unpacked(); } /** \brief Unpacks a block of data from the current received buffer. @@ -112,22 +110,22 @@ bool PCU_Comm_Unpacked2(PCUHandle h) { PCU_Comm_Unpacked can help with this. */ int PCU_Comm_Unpack2(PCUHandle h, void *data, size_t size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Unpack called before Comm_Init"); - return static_cast(h->ptr)->Unpack(data, size); + return static_cast(h.ptr)->Unpack(data, size); } void PCU_Comm_Order2(PCUHandle h, bool on) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Order called before Comm_Init"); - static_cast(h->ptr)->Order(on); + static_cast(h.ptr)->Order(on); } /** \brief Blocking barrier over all threads. */ void PCU_Barrier2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Barrier called before Comm_Init"); - static_cast(h->ptr)->Barrier(); + static_cast(h.ptr)->Barrier(); } /** \brief Performs an Allreduce sum of double arrays. @@ -137,113 +135,113 @@ void PCU_Barrier2(PCUHandle h) { given by each rank. */ void PCU_Add_Doubles2(PCUHandle h, double *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_Doubles called before Comm_Init"); - static_cast(h->ptr)->Add(p, n); + static_cast(h.ptr)->Add(p, n); } double PCU_Add_Double2(PCUHandle h, double x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_Double called before Comm_Init"); - return static_cast(h->ptr)->Add(x); + return static_cast(h.ptr)->Add(x); } /** \brief Performs an Allreduce minimum of double arrays. */ void PCU_Min_Doubles2(PCUHandle h, double *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Min_Doubles called before Comm_Init"); - static_cast(h->ptr)->Min(p, n); + static_cast(h.ptr)->Min(p, n); } double PCU_Min_Double2(PCUHandle h, double x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Min_Double called before Comm_Init"); - return static_cast(h->ptr)->Min(x); + return static_cast(h.ptr)->Min(x); } /** \brief Performs an Allreduce maximum of double arrays. */ -void PCU_Max_Doubles2(PCUHandle, double *p, size_t n) { - if (h->ptr == nullptr) +void PCU_Max_Doubles2(PCUHandle h, double *p, size_t n) { + if (h.ptr == nullptr) reel_fail("Max_Doubles called before Comm_Init"); - static_cast(h->ptr)->Max(p, n); + static_cast(h.ptr)->Max(p, n); } double PCU_Max_Double2(PCUHandle h, double x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_Double called before Comm_Init"); - return static_cast(h->ptr)->Max(x); + return static_cast(h.ptr)->Max(x); } /** \brief Performs an Allreduce sum of integers */ void PCU_Add_Ints2(PCUHandle h, int *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_Ints called before Comm_Init"); - static_cast(h->ptr)->Add(p, n); + static_cast(h.ptr)->Add(p, n); } int PCU_Add_Int2(PCUHandle h, int x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_Int called before Comm_Init"); - return static_cast(h->ptr)->Add(x); + return static_cast(h.ptr)->Add(x); } /** \brief Performs an Allreduce sum of long integers */ void PCU_Add_Longs2(PCUHandle h, long *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_Longs called before Comm_Init"); - static_cast(h->ptr)->Add(p, n); + static_cast(h.ptr)->Add(p, n); } long PCU_Add_Long2(PCUHandle h, long x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_Long called before Comm_Init"); - return static_cast(h->ptr)->Add(x); + return static_cast(h.ptr)->Add(x); } /** \brief Performs an Allreduce sum of size_t unsigned integers */ void PCU_Add_SizeTs2(PCUHandle h, size_t *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_SizeTs called before Comm_Init"); - static_cast(h->ptr)->Add(p, n); + static_cast(h.ptr)->Add(p, n); } size_t PCU_Add_SizeT2(PCUHandle h, size_t x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Add_SizeT called before Comm_Init"); - return static_cast(h->ptr)->Add(x); + return static_cast(h.ptr)->Add(x); } /** \brief Performs an Allreduce minimum of size_t unsigned integers */ void PCU_Min_SizeTs2(PCUHandle h, size_t *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Min_SizeTs called before Comm_Init"); - static_cast(h->ptr)->Min(p, n); + static_cast(h.ptr)->Min(p, n); } size_t PCU_Min_SizeT2(PCUHandle h, size_t x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Min_SizeT called before Comm_Init"); - return static_cast(h->ptr)->Min(x); + return static_cast(h.ptr)->Min(x); } /** \brief Performs an Allreduce maximum of size_t unsigned integers */ void PCU_Max_SizeTs2(PCUHandle h, size_t *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_SizeTs called before Comm_Init"); - static_cast(h->ptr)->Max(p, n); + static_cast(h.ptr)->Max(p, n); } size_t PCU_Max_SizeT2(PCUHandle h, size_t x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_SizeT called before Comm_Init"); - return static_cast(h->ptr)->Max(x); + return static_cast(h.ptr)->Max(x); } /** \brief Performs an exclusive prefix sum of integer arrays. @@ -253,128 +251,128 @@ size_t PCU_Max_SizeT2(PCUHandle h, size_t x) { given by ranks lower than the calling rank. */ void PCU_Exscan_Ints2(PCUHandle h, int *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Exscan_Ints called before Comm_Init"); - static_cast(h->ptr)->Exscan(p, n); + static_cast(h.ptr)->Exscan(p, n); } int PCU_Exscan_Int2(PCUHandle h, int x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Exscan_Int called before Comm_Init"); - return static_cast(h->ptr)->Exscan(x); + return static_cast(h.ptr)->Exscan(x); } /** \brief See PCU_Exscan_Ints */ void PCU_Exscan_Longs2(PCUHandle h, long *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Exscan_Longs called before Comm_Init"); - static_cast(h->ptr)->Exscan(p, n); + static_cast(h.ptr)->Exscan(p, n); } long PCU_Exscan_Long2(PCUHandle h, long x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Exscan_Long called before Comm_Init"); - return static_cast(h->ptr)->Exscan(x); + return static_cast(h.ptr)->Exscan(x); } /** \brief Performs an Allreduce minimum of int arrays. */ void PCU_Min_Ints2(PCUHandle h, int *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Min_Ints called before Comm_Init"); - static_cast(h->ptr)->Min(p, n); + static_cast(h.ptr)->Min(p, n); } int PCU_Min_Int2(PCUHandle h, int x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Min_Int called before Comm_Init"); - return static_cast(h->ptr)->Min(x); + return static_cast(h.ptr)->Min(x); } /** \brief Performs an Allreduce maximum of int arrays. */ void PCU_Max_Ints2(PCUHandle h, int *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_Ints called before Comm_Init"); - global_pcu->Max(p, n); + static_cast(h.ptr)->Max(p, n); } int PCU_Max_Int2(PCUHandle h, int x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_Int called before Comm_Init"); - return static_cast(h->ptr)->Max(x); + return static_cast(h.ptr)->Max(x); } /** \brief Performs an Allreduce maximum of long arrays. */ void PCU_Max_Longs2(PCUHandle h, long *p, size_t n) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_Longs called before Comm_Init"); - static_cast(h->ptr)->Max(p, n); + static_cast(h.ptr)->Max(p, n); } long PCU_Max_Long2(PCUHandle h, long x) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Max_Long called before Comm_Init"); - return static_cast(h->ptr)->Max(x); + return static_cast(h.ptr)->Max(x); } /** \brief Performs a parallel logical OR reduction */ int PCU_Or2(PCUHandle h, int c) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Or called before Comm_Init"); - return static_cast(h->ptr)->Or(c); + return static_cast(h.ptr)->Or(c); } /** \brief Performs a parallel logical AND reduction */ int PCU_And2(PCUHandle h, int c) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("And called before Comm_Init"); - return static_cast(h->ptr)->And(c); + return static_cast(h.ptr)->And(c); } /** \brief Returns the unique rank of the calling process. */ -int PCU_Proc_Self2(PCUHandle h, void) { - if (h->ptr == nullptr) +int PCU_Proc_Self2(PCUHandle h) { + if (h.ptr == nullptr) reel_fail("Proc_Self called before Comm_Init"); - return static_cast(h->ptr)->Self(); + return static_cast(h.ptr)->Self(); } /** \brief Returns the number of processes. */ -int PCU_Proc_Peers2(PCUHandle h, void) { - if (h->ptr == nullptr) +int PCU_Proc_Peers2(PCUHandle h) { + if (h.ptr == nullptr) reel_fail("Proc_Peers called before Comm_Init"); - return static_cast(h->ptr)->Peers(); + return static_cast(h.ptr)->Peers(); } /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ int PCU_Comm_Rank2(PCUHandle h, int *rank) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Rank called before Comm_Init"); - *rank = static_cast(h->ptr)->Self(); + *rank = static_cast(h.ptr)->Self(); return PCU_SUCCESS; } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ int PCU_Comm_Size2(PCUHandle h, int *size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Size called before Comm_Init"); - *size = static_cast(h->ptr)->Peers(); + *size = static_cast(h.ptr)->Peers(); return PCU_SUCCESS; } /** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized2(PCUHandle h) { return h->ptr != nullptr; } +bool PCU_Comm_Initialized2(PCUHandle h) { return h.ptr != nullptr; } /** \brief Deprecated, see PCU_Comm_Begin. */ int PCU_Comm_Start2(PCUHandle h, PCU_Method method) { (void)method; // warning silencer - static_cast(h->ptr)->Begin(); + static_cast(h.ptr)->Begin(); return PCU_SUCCESS; } @@ -384,9 +382,9 @@ int PCU_Comm_Start2(PCUHandle h, PCU_Method method) { PCU_Comm_Send. */ int PCU_Comm_Packed2(PCUHandle h, int to_rank, size_t *size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Packed called before Comm_Init"); - return static_cast(h->ptr)->Packed(to_rank, size); + return static_cast(h.ptr)->Packed(to_rank, size); } /** \brief Packs a message to be sent to \a to_rank. @@ -399,16 +397,16 @@ int PCU_Comm_Packed2(PCUHandle h, int to_rank, size_t *size) { If this function is used, PCU_Comm_Pack should not be used. */ int PCU_Comm_Write2(PCUHandle h, int to_rank, const void *data, size_t size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Write called before Comm_Init"); - return static_cast(h->ptr)->Write(to_rank, data, size); + return static_cast(h.ptr)->Write(to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ bool PCU_Comm_Receive2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Receive called before Comm_Init"); - return static_cast(h->ptr)->Receive(); + return static_cast(h.ptr)->Receive(); } /** \brief Receives a message for this communication phase. @@ -424,32 +422,32 @@ bool PCU_Comm_Receive2(PCUHandle h) { it is strongly recommended that this data be read and not modified. */ bool PCU_Comm_Read2(PCUHandle h, int *from_rank, void **data, size_t *size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Read called before Comm_Init"); - return static_cast(h->ptr)->Read(from_rank, data, size); + return static_cast(h.ptr)->Read(from_rank, data, size); } void PCU_Debug_Open2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Debug_Open called before Comm_Init"); - static_cast(h->ptr)->DebugOpen(); + static_cast(h.ptr)->DebugOpen(); } /** \brief like fprintf, contents go to debugN.txt */ void PCU_Debug_Print2(PCUHandle h, const char *format, ...) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Debug_Print called before Comm_Init"); va_list arglist; va_start(arglist, format); - static_cast(h->ptr)->DebugPrint(format, arglist); + static_cast(h.ptr)->DebugPrint(format, arglist); va_end(arglist); } /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ int PCU_Comm_From2(PCUHandle h, int *from_rank) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_From called before Comm_Init"); - return static_cast(h->ptr)->From(from_rank); + return static_cast(h.ptr)->From(from_rank); } /** \brief Returns in * \a size the bytes in the current received buffer @@ -458,9 +456,9 @@ int PCU_Comm_From2(PCUHandle h, int *from_rank) { much unpacking has been done. */ int PCU_Comm_Received2(PCUHandle h, size_t *size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Received called before Comm_Init"); - return static_cast(h->ptr)->Received(size); + return static_cast(h.ptr)->Received(size); } /** \brief Extracts a block of data from the current received buffer. @@ -470,9 +468,9 @@ int PCU_Comm_Received2(PCUHandle h, size_t *size) { The returned pointer must not be freed by the user. */ void *PCU_Comm_Extract2(PCUHandle h, size_t size) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Comm_Extract called before Comm_Init"); - return static_cast(h->ptr)->Extract(size); + return static_cast(h.ptr)->Extract(size); } /** \brief Reinitializes PCU with a new MPI communicator. @@ -484,9 +482,9 @@ void *PCU_Comm_Extract2(PCUHandle h, size_t size) { and should be used sparingly. */ void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Switch_Comm called before Comm_Init"); - static_cast(h->ptr)->SwitchMPIComm(new_comm); + static_cast(h.ptr)->SwitchMPIComm(new_comm); } /** \brief Return the current MPI communicator @@ -495,9 +493,9 @@ void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm) { otherwise. */ MPI_Comm PCU_Get_Comm2(PCUHandle h) { - if (h->ptr == nullptr) + if (h.ptr == nullptr) reel_fail("Get_Comm called before Comm_Init"); - return static_cast(h->ptr)->GetMPIComm(); + return static_cast(h.ptr)->GetMPIComm(); } /** \brief Return the time in seconds since some time in the past @@ -511,7 +509,7 @@ double PCU_GetMem2(void) { return pcu::GetMem(); } /** \brief Return the global PCU */ -pcu::PCU* PCU_GetGlobal2(PCUHandle h) { return static_cast(h->ptr); } +pcu::PCU* PCU_GetGlobal2(PCUHandle h) { return static_cast(h.ptr); } From 3101beb6d50d2e077e5e12dacc61a0dabc38a30c Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 30 Jan 2024 15:19:43 -0500 Subject: [PATCH 043/141] re-checking up to ma --- apf_cap/apfCAP.cc | 16 ++++++++-------- crv/crvAdapt.cc | 10 +++++----- crv/crvShape.cc | 28 ++++++++++++++-------------- crv/crvVtk.cc | 4 ++-- dsp/dspGraphDistance.cc | 2 +- ma/ma.cc | 8 ++++---- ma/maAdapt.cc | 2 +- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/apf_cap/apfCAP.cc b/apf_cap/apfCAP.cc index 8e59797e8..a3e72a6d5 100644 --- a/apf_cap/apfCAP.cc +++ b/apf_cap/apfCAP.cc @@ -855,7 +855,7 @@ const char* MeshCAP::getTagName(MeshTag* tag) bool MeshCAP::isShared(MeshEntity* e) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::isShared called in a parallel run!\n"); return false; } @@ -863,7 +863,7 @@ bool MeshCAP::isShared(MeshEntity* e) bool MeshCAP::isOwned(MeshEntity* e) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::isOwned called in a parallel run!\n"); return true; } @@ -871,7 +871,7 @@ bool MeshCAP::isOwned(MeshEntity* e) int MeshCAP::getOwner(MeshEntity* e) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getOwner called in a parallel run!\n"); return 0; } @@ -880,23 +880,23 @@ void MeshCAP::getRemotes(MeshEntity* e, Copies& remotes) { (void)e; (void)remotes; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getRemotes called in a parallel run!\n"); } void MeshCAP::getResidence(MeshEntity* e, Parts& residence) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getResidence called in a parallel run!\n"); residence.insert(0); } int MeshCAP::getId() { - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getId called in a parallel run!\n"); - return PCU_Comm_Self(); + return getPCU()->Self(); } void MeshCAP::migrate(Migration* plan) @@ -909,7 +909,7 @@ void MeshCAP::getMatches(MeshEntity* e, Matches& m) { (void)e; (void)m; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getMatches called in a parallel run!\n"); } diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc index 12d58abb1..a6bd5fa41 100644 --- a/crv/crvAdapt.cc +++ b/crv/crvAdapt.cc @@ -73,14 +73,14 @@ void splitEdges(ma::Adapt* a) static void refine(ma::Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); --(a->refinesLeft); long count = ma::markEdgesToSplit(a); if ( ! count) { return; } splitEdges(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); ma::print("split %li edges in %f seconds",count,t1-t0); } @@ -132,7 +132,7 @@ int markInvalidEntities(Adapt* a) } m->end(it); delete qual; - return PCU_Add_Int(count); + return m->getPCU()->Add(count); } int getTag(Adapt* a, ma::Entity* e) @@ -200,7 +200,7 @@ void adapt(ma::Input* in) in->shapeHandler = crv::getShapeHandler; ma::print("Curved Adaptation Version 2.0 !"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); ma::validateInput(in); Adapt* a = new Adapt(in); ma::preBalance(a); @@ -228,7 +228,7 @@ void adapt(ma::Input* in) cleanupLayer(a); ma::printQuality(a); ma::postBalance(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); ma::print("mesh adapted in %f seconds",t1-t0); apf::printStats(a->mesh); crv::clearTags(a); diff --git a/crv/crvShape.cc b/crv/crvShape.cc index 33f421292..4ac1a99f4 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -635,7 +635,7 @@ static int markEdgesOppLargeAnglesTri(Adapt* a) } m->end(it); } while(count > prev_count); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } static int markEdgesOppLargeAnglesTet(Adapt* a) @@ -663,7 +663,7 @@ static int markEdgesOppLargeAnglesTet(Adapt* a) } m->end(it); } while(count > prev_count); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } /* The whole idea is to do the quality check once, @@ -701,12 +701,12 @@ static int markEdgesToFix(Adapt* a, int flag) } m->end(it); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } int fixLargeBoundaryAngles(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markEdgesOppLargeAnglesTet(a); count += markEdgesOppLargeAnglesTri(a); @@ -714,7 +714,7 @@ int fixLargeBoundaryAngles(Adapt* a) return 0; } splitEdges(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); ma::print("split %d boundary edges with " "large angles in %f seconds",count,t1-t0); return 0; @@ -722,7 +722,7 @@ int fixLargeBoundaryAngles(Adapt* a) static void collapseInvalidEdges(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); ma::Mesh* m = a->mesh; int maxDimension = m->getDimension(); PCU_ALWAYS_ASSERT(checkFlagConsistency(a,1,ma::COLLAPSE)); @@ -733,28 +733,28 @@ static void collapseInvalidEdges(Adapt* a) findIndependentSet(a); successCount += ma::collapseAllEdges(a, modelDimension); } - successCount = PCU_Add_Long(successCount); - double t1 = PCU_Time(); + successCount = m->getPCU()->Add(successCount); + double t1 = pcu::Time(); ma::print("Collapsed %d bad edges " "in %f seconds",successCount, t1-t0); } static void swapInvalidEdges(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); EdgeSwapper es(a); ma::applyOperator(a,&es); - double t1 = PCU_Time(); + double t1 = pcu::Time(); ma::print("Swapped %d bad edges " "in %f seconds",es.ns, t1-t0); } static void repositionInvalidEdges(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); EdgeReshaper es(a); ma::applyOperator(a,&es); - double t1 = PCU_Time(); + double t1 = pcu::Time(); ma::print("Repositioned %d bad edges " "in %f seconds",es.nr, t1-t0); } @@ -827,7 +827,7 @@ void fixCrvElementShapes(Adapt* a) if ( ! a->input->shouldFixShape) return; a->input->shouldForceAdaptation = true; - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markCrvBadQuality(a); int originalCount = count; int prev_count; @@ -850,7 +850,7 @@ void fixCrvElementShapes(Adapt* a) count = markCrvBadQuality(a); ++i; } while(count < prev_count && i < 6); // the second conditions is to make sure this does not take long - double t1 = PCU_Time(); + double t1 = pcu::Time(); ma::print("bad shapes down from %d to %d in %f seconds", originalCount,count,t1-t0); a->input->shouldForceAdaptation = false; diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc index 1d2b352f0..236126f7f 100644 --- a/crv/crvVtk.cc +++ b/crv/crvVtk.cc @@ -1001,7 +1001,7 @@ void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix) void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (!m->getPCU()->Self()) { makeDirectories(prefix, type, n); writePvtuFile(getPvtuDirectoryStr(prefix, type, n).c_str(),"",m,type); @@ -1056,7 +1056,7 @@ void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix) } m->getPCU()->Barrier(); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!m->getPCU()->Self()) lion_oprint(1,"%s vtk files %s written in %f seconds\n", apf::Mesh::typeName[type],getPvtuDirectoryStr(prefix, type, n).c_str(),t1 - t0); diff --git a/dsp/dspGraphDistance.cc b/dsp/dspGraphDistance.cc index a73d3708f..853104f0a 100644 --- a/dsp/dspGraphDistance.cc +++ b/dsp/dspGraphDistance.cc @@ -67,7 +67,7 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, apf::Copies remotes; m->getRemotes(sv, remotes); APF_ITERATE(apf::Copies, remotes, rit) - PCU_COMM_PACK(rit->first, rit->second); + m->getPCU()->Pack(rit->first, rit->second); } m->getPCU()->Send(); while (m->getPCU()->Receive()) { diff --git a/ma/ma.cc b/ma/ma.cc index 64d692e25..ee2f6ac2e 100644 --- a/ma/ma.cc +++ b/ma/ma.cc @@ -24,7 +24,7 @@ namespace ma { void adapt(Input* in) { print("version 2.0 !"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); @@ -51,7 +51,7 @@ void adapt(Input* in) if (in->ownsSolutionTransfer) delete in->solutionTransfer; delete in; - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("mesh adapted in %f seconds",t1-t0); apf::printStats(m); } @@ -64,7 +64,7 @@ void adapt(const Input* in) void adaptVerbose(Input* in, bool verbose) { print("version 2.0 - dev !"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); @@ -125,7 +125,7 @@ void adaptVerbose(Input* in, bool verbose) if (in->ownsSolutionTransfer) delete in->solutionTransfer; delete in; - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("mesh adapted in %f seconds",t1-t0); apf::printStats(m); } diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index f56b9ea4b..eeae6ea06 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -111,7 +111,7 @@ void setFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == a->mesh->getPCU()->Self()); setFlag(a, it->entity, flag); } } From ee1d7e4a823941534887adc7deee96b81815598f Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 30 Jan 2024 16:23:48 -0500 Subject: [PATCH 044/141] ma folder redo --- ma/maAdapt.cc | 10 ++++----- ma/maBalance.cc | 12 +++++------ ma/maCoarsen.cc | 6 +++--- ma/maCrawler.cc | 6 +++--- ma/maExtrude.cc | 8 +++---- ma/maLayer.cc | 12 +++++------ ma/maLayerCoarsen.cc | 12 +++++------ ma/maLayerRefine.cc | 8 +++---- ma/maLayerSnap.cc | 50 ++++++++++++++++++++++---------------------- ma/maMatch.cc | 6 +++--- ma/maMesh.cc | 10 ++++----- pcu/PCU2.h | 6 ------ pcu/pcu2.cc | 8 +------ 13 files changed, 71 insertions(+), 83 deletions(-) diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index eeae6ea06..ec64bfcd3 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -131,7 +131,7 @@ void clearFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == a->mesh->getPCU()->Self()); clearFlag(a, it->entity, flag); } } @@ -238,8 +238,8 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) continue; bool value = getFlag(a, e, flag); APF_ITERATE(apf::CopyArray, others, rit) { - PCU_COMM_PACK(rit->peer, rit->entity); - PCU_COMM_PACK(rit->peer, value); + m->getPCU()->Pack(rit->peer, rit->entity); + m->getPCU()->Pack(rit->peer, value); } } m->end(it); @@ -321,7 +321,7 @@ long markEntities( setFlag(a,e,setFalseFlag); } m->end(it); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } void NewEntities::reset() @@ -509,7 +509,7 @@ void syncFlag(Adapt* a, int dimension, int flag) apf::CopyArray others; sh->getCopies(e, others); APF_ITERATE(apf::CopyArray, others, rit) - PCU_COMM_PACK(rit->peer, rit->entity); + m->getPCU()->Pack(rit->peer, rit->entity); } m->end(it); m->getPCU()->Send(); diff --git a/ma/maBalance.cc b/ma/maBalance.cc index edeefa96c..7fdefe990 100644 --- a/ma/maBalance.cc +++ b/ma/maBalance.cc @@ -135,7 +135,7 @@ double estimateWeightedImbalance(Adapt* a) void preBalance(Adapt* a) { - if (PCU_Comm_Peers()==1) + if (a->mesh->getPCU()->Peers()==1) return; Input* in = a->input; // First take care of user overrides. That is, if any of the three options @@ -164,7 +164,7 @@ void preBalance(Adapt* a) #ifdef PUMI_HAS_ZOLTAN // The parmetis multi-level graph partitioner memory usage grows // significantly with process count beyond 16K processes - if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) { + if (a->mesh->getPCU()->Peers() < MAX_ZOLTAN_GRAPH_RANKS) { runZoltan(a); return; } @@ -181,7 +181,7 @@ void preBalance(Adapt* a) void midBalance(Adapt* a) { - if (PCU_Comm_Peers()==1) + if (a->mesh->getPCU()->Peers()==1) return; Input* in = a->input; // First take care of user overrides. That is, if any of the three options @@ -204,7 +204,7 @@ void midBalance(Adapt* a) #ifdef PUMI_HAS_ZOLTAN // The parmetis multi-level graph partitioner memory usage grows // significantly with process count beyond 16K processes - if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) { + if (a->mesh->getPCU()->Peers() < MAX_ZOLTAN_GRAPH_RANKS) { runZoltan(a); return; } @@ -221,7 +221,7 @@ void midBalance(Adapt* a) void postBalance(Adapt* a) { - if (PCU_Comm_Peers()==1) + if (a->mesh->getPCU()->Peers()==1) return; Input* in = a->input; // First take care of user overrides. That is, if any of the three options @@ -252,7 +252,7 @@ void postBalance(Adapt* a) #ifdef PUMI_HAS_ZOLTAN // The parmetis multi-level graph partitioner memory usage grows // significantly with process count beyond 16K processes - if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) { + if (a->mesh->getPCU()->Peers() < MAX_ZOLTAN_GRAPH_RANKS) { runZoltan(a); printEntityImbalance(a->mesh); return; diff --git a/ma/maCoarsen.cc b/ma/maCoarsen.cc index 26379178c..419393946 100644 --- a/ma/maCoarsen.cc +++ b/ma/maCoarsen.cc @@ -230,7 +230,7 @@ bool coarsen(Adapt* a) { if (!a->input->shouldCoarsen) return false; - double t0 = PCU_Time(); + double t0 = pcu::Time(); --(a->coarsensLeft); long count = markEdgesToCollapse(a); if ( ! count) @@ -248,8 +248,8 @@ bool coarsen(Adapt* a) else successCount += collapseAllEdges(a, modelDimension); } - successCount = PCU_Add_Long(successCount); - double t1 = PCU_Time(); + successCount = m->getPCU()->Add(successCount); + double t1 = pcu::Time(); print("coarsened %li edges in %f seconds",successCount,t1-t0); return true; } diff --git a/ma/maCrawler.cc b/ma/maCrawler.cc index 2bac53d10..79b66e8e6 100644 --- a/ma/maCrawler.cc +++ b/ma/maCrawler.cc @@ -16,7 +16,7 @@ void syncLayer(Crawler* c, Crawler::Layer& layer) apf::Copies remotes; m->getRemotes(e,remotes); APF_ITERATE(apf::Copies,remotes,it) { - PCU_COMM_PACK(it->first,it->second); + m->getPCU()->Pack(it->first,it->second); c->send(e, it->first); } } @@ -49,7 +49,7 @@ void crawlLayers(Crawler* c) { Crawler::Layer layer; c->begin(layer); - while (PCU_Or( ! layer.empty())) { + while (c->mesh->getPCU()->Or( ! layer.empty())) { crawlLayer(c, layer); syncLayer(c, layer); } @@ -157,7 +157,7 @@ struct LayerNumberer : public Crawler void send(Entity* v, int to) { int n = t.getNumber(v); - PCU_COMM_PACK(to, n); + m->getPCU()->Pack(to, n); } bool recv(Entity* v, int) { diff --git a/ma/maExtrude.cc b/ma/maExtrude.cc index 2ee63f3ff..acaade12c 100644 --- a/ma/maExtrude.cc +++ b/ma/maExtrude.cc @@ -303,8 +303,8 @@ void stitchVerts(Mesh* m, Crawler::Layer const& prev_verts, APF_ITERATE(Remotes, remotes, rit) { int remote_part = rit->first; Entity* remote_prev_vert = rit->second; - PCU_COMM_PACK(remote_part, remote_prev_vert); - PCU_COMM_PACK(remote_part, next_vert); + m->getPCU()->Pack(remote_part, remote_prev_vert); + m->getPCU()->Pack(remote_part, next_vert); } } m->getPCU()->Send(); @@ -312,8 +312,8 @@ void stitchVerts(Mesh* m, Crawler::Layer const& prev_verts, int remote_part = m->getPCU()->Sender(); Entity* prev_vert; Entity* remote_next_vert; - PCU_COMM_UNPACK(prev_vert); - PCU_COMM_UNPACK(remote_next_vert); + m->getPCU()->Unpack(prev_vert); + m->getPCU()->Unpack(remote_next_vert); int prev_idx; m->getIntTag(prev_vert, indices, &prev_idx); Entity* next_vert = next_verts.at(prev_idx); diff --git a/ma/maLayer.cc b/ma/maLayer.cc index bb214e8be..5c2bd1191 100644 --- a/ma/maLayer.cc +++ b/ma/maLayer.cc @@ -38,7 +38,7 @@ static long markLayerElements(Adapt* a) } } } - n = PCU_Add_Long(n); + n = m->getPCU()->Add(n); a->hasLayer = (n != 0); if ( ! a->hasLayer) return 0; @@ -94,12 +94,12 @@ void unfreezeLayer(Adapt* a) void resetLayer(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); long n = markLayerElements(a); if (!n) return; freezeLayer(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("marked %ld layer elements in %f seconds", n, t1 - t0); } @@ -166,7 +166,7 @@ void collectForLayerRefine(Refine* r) void checkLayerShape(Mesh* m, const char* key) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Iterator* it = m->begin(m->getDimension()); Entity* e; long n = 0; @@ -187,8 +187,8 @@ void checkLayerShape(Mesh* m, const char* key) ++n; } m->end(it); - n = PCU_Add_Long(n); - double t1 = PCU_Time(); + n = m->getPCU()->Add(n); + double t1 = pcu::Time(); print("%s: checked layer quality in %f seconds: %ld unsafe elements", key, t1 - t0, n); } diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index b6611eb33..595a84312 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -41,7 +41,7 @@ static long markBaseEdgesToCollapse(Adapt* a) } } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } struct CurveLocalizer : public Crawler @@ -134,12 +134,12 @@ struct CurveLocalizer : public Crawler void send(Entity* v, int to) { int dest = getVertDest(v); - PCU_COMM_PACK(to, dest); + m->getPCU()->Pack(to, dest); } bool recv(Entity* v, int) { int dest; - PCU_COMM_UNPACK(dest); + m->getPCU()->Unpack(dest); return handle(v, dest); } Adapt* a; @@ -259,7 +259,7 @@ static long collapseAllStacks(Adapt* a, int d) allSuccesses += successCount; ++round; } while (a->mesh->getPCU()->Or(skipCount)); - return PCU_Add_Long(allSuccesses); + return a->mesh->getPCU()->Add(allSuccesses); } bool coarsenLayer(Adapt* a) @@ -268,7 +268,7 @@ bool coarsenLayer(Adapt* a) return false; if ( ! a->input->shouldCoarsenLayer) return false; - double t0 = PCU_Time(); + double t0 = pcu::Time(); allowLayerToCollapse(a); findLayerBase(a); long count = markBaseEdgesToCollapse(a); @@ -283,7 +283,7 @@ bool coarsenLayer(Adapt* a) findIndependentSet(a); successCount += collapseAllStacks(a, d); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("coarsened %li layer edges in %f seconds",successCount,t1-t0); resetLayer(a); return true; diff --git a/ma/maLayerRefine.cc b/ma/maLayerRefine.cc index a8b13e6e2..004e13463 100644 --- a/ma/maLayerRefine.cc +++ b/ma/maLayerRefine.cc @@ -73,12 +73,12 @@ struct SplitTagger : public Crawler void send(Entity* e, int to) { bool has = getFlag(a, e, SPLIT); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* e, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); if (getFlag(a, e, CHECKED)) return false; handle(e, has); @@ -209,12 +209,12 @@ struct Disambiguator : public Crawler void send(Entity* t, int to) { int diag = getDiagonalFromFlag(a, t); - PCU_COMM_PACK(to, diag); + m->getPCU()->Pack(to, diag); } bool recv(Entity* t, int) { int diag; - PCU_COMM_UNPACK(diag); + m->getPCU()->Unpack(diag); if (getFlag(a, t, CHECKED)) return false; setFlag(a, t, CHECKED); diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index 0c64e6448..09840318f 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -58,20 +58,20 @@ struct SnapTagger : public Crawler void send(Entity* v, int to) { bool has = m->hasTag(v, snapTag); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); if (has) { Vector s; m->getDoubleTag(v, snapTag, &s[0]); - PCU_COMM_PACK(to, s); + m->getPCU()->Pack(to, s); } } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); Vector s; if (has) - PCU_COMM_UNPACK(s); + m->getPCU()->Unpack(s); if (getFlag(a, v, CHECKED)) return false; setFlag(a, v, CHECKED); @@ -160,12 +160,12 @@ struct BaseTopLinker : public Crawler { int link[2]; m->getIntTag(v, linkTag, link); - PCU_COMM_PACK(to, link); + m->getPCU()->Pack(to, link); } bool recv(Entity* v, int) { int link[2]; - PCU_COMM_UNPACK(link); + m->getPCU()->Unpack(link); if (hasLink(v)) return false; m->setIntTag(v, linkTag, link); @@ -234,7 +234,7 @@ struct LayerSnapper : public Crawler } } syncLayer(this, owned); - PCU_Add_Longs(&ncurves, 1); + m->getPCU()->Add(&ncurves, 1); } void end() { @@ -252,12 +252,12 @@ struct LayerSnapper : public Crawler void send(Entity* v, int to) { bool has = m->hasTag(v, snapTag); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); if (getFlag(a, v, CHECKED)) return false; handle(v, has); @@ -271,10 +271,10 @@ struct LayerSnapper : public Crawler static long snapAllCurves(Adapt* a, Tag* snapTag) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); LayerSnapper op(a, snapTag); crawlLayers(&op); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("snapped %ld curves in %f seconds", op.ncurves, t1 - t0); return op.ncurves; } @@ -352,12 +352,12 @@ struct UnsnapChecker : public Crawler void send(Entity* v, int to) { bool has = getFlag(a, v, LAYER_UNSNAP); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); bool wasChecked = getFlag(a, v, CHECKED); if (wasChecked) { if (has) @@ -397,11 +397,11 @@ static void crawlLayers_doubleSync(Crawler* c) static bool checkForUnsnap(Adapt* a, Tag* snapTag) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); UnsnapChecker op(a, snapTag); crawlLayers_doubleSync(&op); bool notOk = a->mesh->getPCU()->Or(op.foundAnything); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (notOk) print("checked snapped curves in %f seconds, found some to unsnap", t1 - t0); else @@ -425,19 +425,19 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) m->isOwned(v)) { int peer, link; l.getLink(v, peer, link); - PCU_COMM_PACK(peer, link); + m->getPCU()->Pack(peer, link); ++n; } m->end(it); m->getPCU()->Send(); while (m->getPCU()->Receive()) { int link; - PCU_COMM_UNPACK(link); + m->getPCU()->Unpack(link); Entity* v = l.lookup(link); setFlag(a, v, LAYER_UNSNAP); PCU_ALWAYS_ASSERT(m->hasTag(v, snapTag)); } - n = PCU_Add_Long(n); + n = m->getPCU()->Add(n); print("fed back unsnap flag from %ld tops", n); } @@ -486,7 +486,7 @@ struct Unsnapper : public Crawler ++ncurves; } } - PCU_Add_Longs(&ncurves, 1); + m->getPCU()->Add(&ncurves, 1); syncLayer(this, owned); } void end() @@ -506,12 +506,12 @@ struct Unsnapper : public Crawler void send(Entity* v, int to) { bool has = getFlag(a, v, LAYER_UNSNAP); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); if (getFlag(a, v, CHECKED)) return false; handle(v, has); @@ -525,10 +525,10 @@ struct Unsnapper : public Crawler static long unsnapMarkedCurves(Adapt* a, Tag* snapTag) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Unsnapper op(a, snapTag); crawlLayers(&op); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("unsnapped %ld curves in %f seconds", op.ncurves, t1 - t0); return op.ncurves; } @@ -537,7 +537,7 @@ void snapLayer(Adapt* a, Tag* snapTag) { if ( ! a->hasLayer) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); findLayerBase(a); tagLayerForSnap(a, snapTag); flagLayerTop(a); @@ -550,7 +550,7 @@ void snapLayer(Adapt* a, Tag* snapTag) nunsnapped += unsnapMarkedCurves(a, snapTag); } delete l; - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("finished snapping %ld of %ld layer curves in %f seconds", nsnapped - nunsnapped, nsnapped, t1 - t0); } diff --git a/ma/maMatch.cc b/ma/maMatch.cc index 0043ca05d..e62758cf4 100644 --- a/ma/maMatch.cc +++ b/ma/maMatch.cc @@ -49,7 +49,7 @@ void matchNewElements(Refine* r) { int to = matches[i].peer; Entity* match = matches[i].entity; - PCU_COMM_PACK(to,match); + m->getPCU()->Pack(to,match); packSplits(to,splits); } } @@ -60,7 +60,7 @@ void matchNewElements(Refine* r) while ( ! m->getPCU()->Unpacked()) { Entity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int number; m->getIntTag(e,r->numberTag,&number); EntityArray& splits = r->newEntities[d][number]; @@ -72,7 +72,7 @@ void matchNewElements(Refine* r) } } } - face_count = PCU_Add_Long(face_count); + face_count = m->getPCU()->Add(face_count); print("updated matching for %li faces",face_count); } diff --git a/ma/maMesh.cc b/ma/maMesh.cc index 7d9b5876f..108f71838 100644 --- a/ma/maMesh.cc +++ b/ma/maMesh.cc @@ -280,8 +280,8 @@ void getBoundingBox(Mesh* m, Vector& lower, Vector& upper) lower.toArray(a); double b[3]; upper.toArray(b); - PCU_Min_Doubles(a, 3); - PCU_Max_Doubles(b, 3); + m->getPCU()->Min(a, 3); + m->getPCU()->Max(b, 3); lower.fromArray(a); upper.fromArray(b); } @@ -303,7 +303,7 @@ Vector getCentroid(Mesh* m) } m->end(it); pointSum.toArray(values); - PCU_Add_Doubles(&(values[0]),4); + m->getPCU()->Add(&(values[0]),4); return Vector(values)/values[3]; } @@ -370,7 +370,7 @@ double getAverageElementSize(Mesh* m) m->end(it); double& count = sums[1]; count = m->count(m->getDimension()); - PCU_Add_Doubles(sums,2); + m->getPCU()->Add(sums,2); return sizeSum / count; } @@ -385,7 +385,7 @@ double getMinimumElementSize(Mesh* m) if (size < minimum) minimum=size; } m->end(it); - return PCU_Min_Double(minimum); + return m->getPCU()->Min(minimum); } void getFaceEdgesAndDirections( diff --git a/pcu/PCU2.h b/pcu/PCU2.h index 2d82507ef..c9da30ddf 100644 --- a/pcu/PCU2.h +++ b/pcu/PCU2.h @@ -97,12 +97,6 @@ int PCU_Comm_Rank2(PCUHandle h, int* rank); int PCU_Comm_Size2(PCUHandle h, int* size); /*deprecated method enum*/ -#ifdef __cplusplus -enum PCU_Method { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD }; -#else -typedef enum { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD } PCU_Method; -#endif -int PCU_Comm_Start2(PCUHandle h, PCU_Method method); /*special MPI_Comm replacement API*/ void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm); diff --git a/pcu/pcu2.cc b/pcu/pcu2.cc index 73b414d42..877ec0e97 100644 --- a/pcu/pcu2.cc +++ b/pcu/pcu2.cc @@ -368,13 +368,7 @@ int PCU_Comm_Size2(PCUHandle h, int *size) { /** \brief Returns true iff PCU has been initialized */ bool PCU_Comm_Initialized2(PCUHandle h) { return h.ptr != nullptr; } -/** \brief Deprecated, see PCU_Comm_Begin. - */ -int PCU_Comm_Start2(PCUHandle h, PCU_Method method) { - (void)method; // warning silencer - static_cast(h.ptr)->Begin(); - return PCU_SUCCESS; -} + /** \brief Returns in * \a size the number of bytes being sent to \a to_rank. \details Returns the size of the buffer being sent to \a to_rank. From f13132ee531f8b17c4b11c6d2ab2d25fd6caa923 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 5 Feb 2024 10:01:39 -0500 Subject: [PATCH 045/141] more fixes --- apf/apfCavityOp.cc | 3 +- ma/maRefine.cc | 8 +-- ma/maShape.cc | 22 ++++---- ma/maSize.cc | 4 +- ma/maSnap.cc | 10 ++-- ma/maTetrahedronize.cc | 14 ++--- mds/apfMDS.cc | 32 +++++------ mds/apfPM.cc | 12 ++-- omega_h/apfOmega_h.cc | 54 +++++++++--------- parma/diffMC/parma_balancer.cc | 4 +- parma/diffMC/parma_centroids.cc | 4 +- parma/diffMC/parma_components.cc | 2 +- parma/diffMC/parma_dcpart.cc | 4 +- parma/diffMC/parma_dcpartFixer.cc | 12 ++-- parma/diffMC/parma_edgeEqVtxSelector.cc | 8 +-- parma/diffMC/parma_elmBalancer.cc | 2 +- parma/diffMC/parma_elmLtVtxEdgeSelector.cc | 16 +++--- parma/diffMC/parma_entWeights.cc | 8 +-- parma/diffMC/parma_ghostWeights.cc | 4 +- parma/diffMC/parma_graphDist.cc | 24 ++++---- parma/diffMC/parma_ltSelector.cc | 8 +-- parma/diffMC/parma_step.cc | 6 +- parma/diffMC/parma_vtxBalancer.cc | 2 +- parma/diffMC/parma_vtxPtnWriter.cc | 6 +- parma/diffMC/parma_vtxSelector.cc | 2 +- parma/parma.cc | 64 +++++++++++----------- pcu/PCUObj.h | 4 +- 27 files changed, 170 insertions(+), 169 deletions(-) diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index c13c0c1d8..9ad4d9637 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -166,7 +166,8 @@ bool CavityOp::sendPullRequests(std::vector& received) request.to = mesh->getPCU()->Sender(); while ( ! mesh->getPCU()->Unpacked()) { - PCU_COMM_UNPACK(request.e); + mesh->getPCU()->Unpack(request.e); + //mesh->getPCU()->Unpack(&(request.e), sizeof(request.e)); received.push_back(request); } } diff --git a/ma/maRefine.cc b/ma/maRefine.cc index 2e908f6ce..eddd0180e 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -267,7 +267,7 @@ static void linkNewVerts(Refine* r) { int to = rp->first; message.parent = rp->second; - PCU_COMM_PACK(to,message); + m->getPCU()->Pack(to,message); } } m->getPCU()->Send(); @@ -276,7 +276,7 @@ static void linkNewVerts(Refine* r) int from = m->getPCU()->Sender(); while ( ! m->getPCU()->Unpacked()) { - PCU_COMM_UNPACK(message); + m->getPCU()->Unpack(message); Entity* v = findSplitVert(r,message.parent); m->addRemote(v,from,message.vert); } @@ -419,7 +419,7 @@ void cleanupAfter(Refine* r) bool refine(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); --(a->refinesLeft); setupLayerForSplit(a); long count = markEdgesToSplit(a); @@ -438,7 +438,7 @@ bool refine(Adapt* a) processNewElements(r); destroySplitElements(r); forgetNewEntities(r); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("refined %li edges in %f seconds",count,t1-t0); resetLayer(a); if (a->hasLayer) diff --git a/ma/maShape.cc b/ma/maShape.cc index c3479a07c..3e59cbeb5 100644 --- a/ma/maShape.cc +++ b/ma/maShape.cc @@ -166,7 +166,7 @@ double getMinQuality(Adapt* a) minqual = qual; } m->end(it); - return PCU_Min_Double(minqual); + return m->getPCU()->Min(minqual); } class ShortEdgeFixer : public Operator @@ -740,10 +740,10 @@ class QualityImprover2D : public Operator static double fixShortEdgeElements(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); ShortEdgeFixer fixer(a); applyOperator(a,&fixer); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -779,12 +779,12 @@ static void improveQualities2D(Adapt* a) static double fixLargeAngles(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (a->mesh->getDimension()==3) fixLargeAngleTets(a); else fixLargeAngleTris(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -798,12 +798,12 @@ static void alignLargeAngles(Adapt* a) double improveQualities(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (a->mesh->getDimension() == 3) return 0; // TODO: implement this for 3D else improveQualities2D(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -811,7 +811,7 @@ void fixElementShapes(Adapt* a) { if ( ! a->input->shouldFixShape) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markBadQuality(a); int originalCount = count; int prev_count; @@ -843,7 +843,7 @@ void fixElementShapes(Adapt* a) ((double) prev_count - (double) count) / (double) prev_count); iter++; } while(count < prev_count); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("bad shapes down from %d to %d in %f seconds", originalCount,count,t1-t0); } @@ -853,7 +853,7 @@ void alignElements(Adapt* a) int max_iter = 5; if ( ! a->input->shouldFixShape) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markBadQuality(a); int originalCount = count; int prev_count; @@ -869,7 +869,7 @@ void alignElements(Adapt* a) unMarkBadQuality(a); } while(count < prev_count && i < max_iter); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("non-aligned elements down from %d to %d in %f seconds", originalCount,count,t1-t0); } diff --git a/ma/maSize.cc b/ma/maSize.cc index 1dea70eab..9fdf6ed30 100644 --- a/ma/maSize.cc +++ b/ma/maSize.cc @@ -667,7 +667,7 @@ double getAverageEdgeLength(Mesh* m) edge_count += 1.0; } m->end(it); - PCU_Add_Doubles(sums,2); + m->getPCU()->Add(sums,2); return length_sum / edge_count; } @@ -687,7 +687,7 @@ double getMaximumEdgeLength(Mesh* m, SizeField* sf) maxLength = length; } m->end(it); - PCU_Max_Doubles(&maxLength,1); + m->getPCU()->Max(&maxLength,1); return maxLength; } diff --git a/ma/maSnap.cc b/ma/maSnap.cc index 9f0f5a4cf..73dc2981f 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -780,7 +780,7 @@ bool snapAllVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) { SnapAll op(a, t, isSimple); applyOperator(a, &op); - successCount += PCU_Add_Long(op.successCount); + successCount += a->mesh->getPCU()->Add(op.successCount); return a->mesh->getPCU()->Or(op.didAnything); } @@ -831,7 +831,7 @@ bool snapMatchedVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) { SnapMatched op(a, t, isSimple); applyOperator(a, &op); - successCount += PCU_Add_Long(op.successCount); + successCount += a->mesh->getPCU()->Add(op.successCount); return a->mesh->getPCU()->Or(op.didAnything); } @@ -857,7 +857,7 @@ long tagVertsToSnap(Adapt* a, Tag*& t) ++n; } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } static void markVertsToSnap(Adapt* a, Tag* t) @@ -902,7 +902,7 @@ void snap(Adapt* a) { if ( ! a->input->shouldSnap) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); Tag* tag; /* we are starting to support a few operations on matched meshes, including snapping+UR. this should prevent snapping @@ -913,7 +913,7 @@ void snap(Adapt* a) snapLayer(a, tag); apf::removeTagFromDimension(a->mesh, tag, 0); a->mesh->destroyTag(tag); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("snapped in %f seconds: %ld targets, %ld non-layer snaps", t1 - t0, targets, success); if (a->hasLayer) diff --git a/ma/maTetrahedronize.cc b/ma/maTetrahedronize.cc index b497e3201..391456df6 100644 --- a/ma/maTetrahedronize.cc +++ b/ma/maTetrahedronize.cc @@ -153,12 +153,12 @@ struct QuadFlagger : public Crawler void send(Entity* e, int to) { int diagonal = getDiagonalFromFlag(adapter, e); - PCU_COMM_PACK(to, diagonal); + mesh->getPCU()->Pack(to, diagonal); } bool recv(Entity* e, int) { int diagonal; - PCU_COMM_UNPACK(diagonal); + mesh->getPCU()->Unpack(diagonal); if (getFlag(adapter, e, DIAGONAL_1 | DIAGONAL_2)) return false; setFlag(adapter, e, getFlagFromDiagonal(diagonal)); @@ -498,12 +498,12 @@ void tetrahedronize(Adapt* a) if ( ! a->input->shouldTurnLayerToTets) return; PCU_ALWAYS_ASSERT(a->hasLayer); - double t0 = PCU_Time(); + double t0 = pcu::Time(); prepareLayerToTets(a); Refine* r = a->refine; addAllLayerElements(r); tetrahedronizeCommon(r); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("boundary layer converted to tets in %f seconds",t1-t0); } @@ -620,7 +620,7 @@ static long markIslandPyramids(Adapt* a) } } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } static int countEntitiesWithFlag(Adapt* a, int flag, int dim) @@ -673,7 +673,7 @@ void cleanupLayer(Adapt* a) return; if (!a->input->shouldCleanupLayer) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); long n = prepareIslandCleanup(a); if (!n) { print("no island pyramids found"); @@ -682,7 +682,7 @@ void cleanupLayer(Adapt* a) Refine* r = a->refine; addIslandPyramids(r); tetrahedronizeCommon(r); - double t1 = PCU_Time(); + double t1 = pcu::Time(); print("tetrahedronized %ld island pyramids in %f seconds", n, t1-t0); } diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 568d50051..23724e57e 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -199,7 +199,7 @@ class MeshMDS : public Mesh2 { init(apf::getLagrange(1)); mesh = mds_read_smb(m, pathname, 0, this); - isMatched = PCU_Or(!mds_net_empty(&mesh->matches)); + isMatched = this->getPCU()->Or(!mds_net_empty(&mesh->matches)); ownsModel = true; } ~MeshMDS() @@ -579,14 +579,14 @@ class MeshMDS : public Mesh2 } int getId() { - return PCU_Comm_Self(); + return this->getPCU()->Self(); } void writeNative(const char* fileName) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); mesh = mds_write_smb(mesh, fileName, 0, this); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!this->getPCU()->Self()) lion_oprint(1,"mesh %s written in %f seconds\n", fileName, t1 - t0); } void destroyNative() @@ -929,14 +929,14 @@ Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile) Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Mesh2* m = new MeshMDS(model, meshfile); initResidence(m, m->getDimension()); stitchMesh(m); m->acceptChanges(); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh %s loaded in %f seconds\n", meshfile, PCU_Time() - t0); + if (!m->getPCU()->Self()) + lion_oprint(1,"mesh %s loaded in %f seconds\n", meshfile, pcu::Time() - t0); printStats(m); warnAboutEmptyParts(m); return m; @@ -944,18 +944,18 @@ Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile) Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); static gmi_model* model; model = gmi_load(modelfile); if (!PCU_Comm_Self()) - lion_oprint(1,"model %s loaded in %f seconds\n", modelfile, PCU_Time() - t0); + lion_oprint(1,"model %s loaded in %f seconds\n", modelfile, pcu::Time() - t0); return loadMdsMesh(model, meshfile); } void reorderMdsMesh(Mesh2* mesh, MeshTag* t) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); MeshMDS* m = static_cast(mesh); mds_tag* vert_nums; if (t) { @@ -966,12 +966,12 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) } m->mesh = mds_reorder(m->mesh, 0, vert_nums); if (!m->getPCU()->Self()) - lion_oprint(1,"mesh reordered in %f seconds\n", PCU_Time()-t0); + lion_oprint(1,"mesh reordered in %f seconds\n", pcu::Time()-t0); } Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); int self = PCU_Comm_Self(); int outputPartCount = PCU_Comm_Peers(); apf::Expand expand(inputPartCount, outputPartCount); @@ -999,7 +999,7 @@ Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) } PCU_ALWAYS_ASSERT(m != 0); apf::remapPartition(m, expand); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!PCU_Comm_Self()) lion_oprint(1,"mesh expanded from %d to %d parts in %f seconds\n", inputPartCount, outputPartCount, t1 - t0); @@ -1010,11 +1010,11 @@ Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor) { m = expandMdsMesh(m, g, PCU_Comm_Peers() / factor); - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (PCU_Comm_Self() % factor != 0) plan = new apf::Migration(m, m->findTag("apf_migrate")); m->migrate(plan); - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!PCU_Comm_Self()) lion_oprint(1,"mesh migrated from %d to %d in %f seconds\n", PCU_Comm_Peers() / factor, diff --git a/mds/apfPM.cc b/mds/apfPM.cc index f15704af6..201bb1e55 100644 --- a/mds/apfPM.cc +++ b/mds/apfPM.cc @@ -95,14 +95,14 @@ static void getCountMap(apf::Mesh* m, PM& ps, CountMap& mp) size_t n; getAdjacentParts(m, ps, peers); n = m->count(m->getDimension()); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(apf::Parts, peers, it) - PCU_COMM_PACK(*it, n); - PCU_Comm_Send(); + m->getPCU()->Pack(*it, n); + m->getPCU()->Send(); mp[m->getId()] = n; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(n); - mp[PCU_Comm_Sender()] = n; + while (m->getPCU()->Listen()) { + m->getPCU()->Unpack(n); + mp[m->getPCU()->Sender()] = n; } } diff --git a/omega_h/apfOmega_h.cc b/omega_h/apfOmega_h.cc index f854b7cc3..1c2ada2c9 100644 --- a/omega_h/apfOmega_h.cc +++ b/omega_h/apfOmega_h.cc @@ -125,7 +125,7 @@ static void field_to_osh(osh::Mesh* om, apf::Field* f) { ) { ent_dim = dim; } else { - if (!PCU_Comm_Self()) { + if (!am->getPCU()->Self()) { lion_oprint(1,"not copying field %s to Omega_h\n",name.c_str()); } return; @@ -181,7 +181,7 @@ static void field_from_osh(apf::Mesh* am, osh::Tag const* tag, if (ent_dim == 0) shape = apf::getLagrange(1); else if (ent_dim == dim) shape = apf::getIPFitShape(dim, 1); else { - if (!PCU_Comm_Self()) { + if (!am->getPCU()->Self()) { lion_oprint(1,"not copying field %s to Omega_h\n",name.c_str()); } return; @@ -293,7 +293,7 @@ static void globals_to_osh( } void to_omega_h(osh::Mesh* om, apf::Mesh* am) { - auto comm_mpi = PCU_Get_Comm(); + auto comm_mpi = am->getPCU()->GetMPIComm(); decltype(comm_mpi) comm_impl; MPI_Comm_dup(comm_mpi, &comm_impl); auto comm_osh = osh::CommPtr(new osh::Comm(om->library(), comm_impl)); @@ -385,56 +385,56 @@ static void owners_from_osh( am->setIntTag(ents[i], own_tag, &tmp); } } - PCU_Comm_Begin(); + am->getPCU()->Begin(); for (int i = 0; i < om->nents(ent_dim); ++i) { osh::LO tmp = own_ids[i]; - PCU_COMM_PACK(own_ranks[i], tmp); - PCU_COMM_PACK(own_ranks[i], ents[i]); + am->getPCU()->Pack(own_ranks[i], tmp); + am->getPCU()->(own_ranks[i], ents[i]); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + am->getPCU()->Send(); + while (am->getPCU()->Receive()) { int own_id; - PCU_COMM_UNPACK(own_id); + am->getPCU()->Unpack(own_id); apf::MeshEntity* r; - PCU_COMM_UNPACK(r); - int from = PCU_Comm_Sender(); - if (from == PCU_Comm_Self()) { + am->getPCU()->Unpack(r); + int from = am->getPCU()->Sender(); + if (from == am->getPCU()->Self()) { assert((int)from == own_ranks[own_id]); continue; } apf::MeshEntity* owner = ents[own_id]; am->addRemote(owner, from, r); } - PCU_Comm_Begin(); + am->getPCU()->Begin(); for (int i = 0; i < om->nents(ent_dim); ++i) - if (own_ranks[i] == (int)(PCU_Comm_Self())) { + if (own_ranks[i] == (int)(am->getPCU()->Self())) { apf::Copies remotes; am->getRemotes(ents[i], remotes); int ncopies = remotes.size(); - int self = PCU_Comm_Self(); + int self = am->getPCU()->Self(); APF_ITERATE(apf::Copies, remotes, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_COMM_PACK(it->first, ncopies); - PCU_COMM_PACK(it->first, self); - PCU_COMM_PACK(it->first, ents[i]); + am->getPCU()->Pack(it->first, it->second); + am->getPCU()->Pack(it->first, ncopies); + am->getPCU()->Pack(it->first, self); + am->getPCU()->Pack(it->first, ents[i]); APF_ITERATE(apf::Copies, remotes, it2) if (it2->first != it->first) { - PCU_COMM_PACK(it->first, it2->first); - PCU_COMM_PACK(it->first, it2->second); + am->getPCU()->Pack(it->first, it2->first); + am->getPCU()->Pack(it->first, it2->second); } } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + am->getPCU()->Send(); + while (am->getPCU()->Receive()) { apf::MeshEntity* e; - PCU_COMM_UNPACK(e); + am->getPCU()->Unpack(e); int ncopies; - PCU_COMM_UNPACK(ncopies); + am->getPCU()->Unpack(ncopies); for (int i = 0; i < ncopies; ++i) { int p; - PCU_COMM_UNPACK(p); + am->getPCU()->Unpack(p); apf::MeshEntity* r; - PCU_COMM_UNPACK(r); + am->getPCU()->Unpack(r); am->addRemote(e, p, r); } } diff --git a/parma/diffMC/parma_balancer.cc b/parma/diffMC/parma_balancer.cc index 5e408fa28..3a3eb9fdd 100644 --- a/parma/diffMC/parma_balancer.cc +++ b/parma/diffMC/parma_balancer.cc @@ -35,9 +35,9 @@ namespace parma { void Balancer::balance(apf::MeshTag* wtag, double tolerance) { if( 1 == mesh->getPCU()->Peers() ) return; int step = 0; - double t0 = PCU_Time(); + double t0 = pcu::Time(); while (runStep(wtag,tolerance) && step++ < maxStep); - printTiming(name, step, tolerance, PCU_Time()-t0); + printTiming(name, step, tolerance, pcu::Time()-t0); } void Balancer::monitorUpdate(double v, Slope* s, Average* a) { s->push(v); diff --git a/parma/diffMC/parma_centroids.cc b/parma/diffMC/parma_centroids.cc index d7ba67a35..482e31a5d 100644 --- a/parma/diffMC/parma_centroids.cc +++ b/parma/diffMC/parma_centroids.cc @@ -48,12 +48,12 @@ namespace parma { const Sides::Item* side; s->begin(); while( (side = s->iterate()) ) - PCU_COMM_PACK(side->first, centroid); + m->getPCU()->Pack(side->first, centroid); s->end(); m->getPCU()->Send(); while (m->getPCU()->Listen()) { apf::Vector3 otherCentroid; - PCU_COMM_UNPACK(otherCentroid); + m->getPCU()->Unpack(otherCentroid); set(m->getPCU()->Sender(), otherCentroid); } } diff --git a/parma/diffMC/parma_components.cc b/parma/diffMC/parma_components.cc index 59725f4b2..4f093587c 100644 --- a/parma/diffMC/parma_components.cc +++ b/parma/diffMC/parma_components.cc @@ -148,7 +148,7 @@ namespace parma { for(unsigned i=0; igetPCU()->DebugPrint("core %u is empty... assigning core to bdry\n", i); core[i] = bdry[i]; } } diff --git a/parma/diffMC/parma_dcpart.cc b/parma/diffMC/parma_dcpart.cc index 15005dad9..00d837c4a 100644 --- a/parma/diffMC/parma_dcpart.cc +++ b/parma/diffMC/parma_dcpart.cc @@ -100,7 +100,7 @@ void dcPart::reset() { } unsigned dcPart::numDisconnectedComps() { - double t1 = PCU_Time(); + double t1 = pcu::Time(); reset(); unsigned numDc = 0; size_t count = 0; @@ -120,7 +120,7 @@ unsigned dcPart::numDisconnectedComps() { count += sz; } if( verbose ) - parmaCommons::printElapsedTime(__func__, PCU_Time() - t1); + parmaCommons::printElapsedTime(__func__, pcu::Time() - t1); PCU_ALWAYS_ASSERT(numDc+numIso >= 1); return (numDc+numIso)-1; } diff --git a/parma/diffMC/parma_dcpartFixer.cc b/parma/diffMC/parma_dcpartFixer.cc index 5766df221..41f26d7a4 100644 --- a/parma/diffMC/parma_dcpartFixer.cc +++ b/parma/diffMC/parma_dcpartFixer.cc @@ -41,7 +41,7 @@ class dcPartFixer::PartFixer : public dcPart { int totNumDc() { int ndc = TO_INT(numDisconnectedComps()); - return PCU_Add_Int(ndc); + return m->getPCU()->Add(ndc); } void setupPlan(muu& dcCompTgts, apf::Migration* plan) { @@ -64,11 +64,11 @@ class dcPartFixer::PartFixer : public dcPart { * are tagged */ void fix() { - double t1 = PCU_Time(); + double t1 = pcu::Time(); int loop = 0; int ndc = 0; while( (ndc = totNumDc()) && loop++ < 50 ) { - double t2 = PCU_Time(); + double t2 = pcu::Time(); muu dcCompTgts; unsigned maxSz = 0; @@ -85,14 +85,14 @@ class dcPartFixer::PartFixer : public dcPart { setupPlan(dcCompTgts, plan); reset(); - double t3 = PCU_Time(); + double t3 = pcu::Time(); m->migrate(plan); if( ! m->getPCU()->Self() && vb) parmaCommons::status( "loop %d components %d seconds %.3f %.3f\n", - loop, ndc, t3-t2, PCU_Time()-t3); + loop, ndc, t3-t2, pcu::Time()-t3); } - parmaCommons::printElapsedTime(__func__, PCU_Time() - t1); + parmaCommons::printElapsedTime(__func__, pcu::Time() - t1); } }; diff --git a/parma/diffMC/parma_edgeEqVtxSelector.cc b/parma/diffMC/parma_edgeEqVtxSelector.cc index 31e8da6a6..710014376 100644 --- a/parma/diffMC/parma_edgeEqVtxSelector.cc +++ b/parma/diffMC/parma_edgeEqVtxSelector.cc @@ -140,13 +140,13 @@ namespace { mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, sendingVtx, s) - PCU_COMM_PACK(s->first, s->second); + mesh->getPCU()->Pack(s->first, s->second); mesh->getPCU()->Send(); MigrComm incoming; //map double w; while (mesh->getPCU()->Listen()) { - PCU_COMM_UNPACK(w); + mesh->getPCU()->Unpack(w); incoming.insert(Migr(mesh->getPCU()->Sender(),w)); } @@ -167,12 +167,12 @@ namespace { mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, accept, a) - PCU_COMM_PACK(a->first, a->second); + mesh->getPCU()->Pack(a->first, a->second); mesh->getPCU()->Send(); parma::Mid* capacity = new parma::Mid; double outw; while (mesh->getPCU()->Listen()) { - PCU_COMM_UNPACK(outw); + mesh->getPCU()->Unpack(outw); (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; diff --git a/parma/diffMC/parma_elmBalancer.cc b/parma/diffMC/parma_elmBalancer.cc index 222ef59d9..8c2a32efb 100644 --- a/parma/diffMC/parma_elmBalancer.cc +++ b/parma/diffMC/parma_elmBalancer.cc @@ -46,7 +46,7 @@ namespace { apf::Balancer* Parma_MakeElmBalancer(apf::Mesh* m, double stepFactor, int verbosity) { - if( !PCU_Comm_Self() && verbosity ) + if( !m->getPCU()->Self() && verbosity ) status("stepFactor %.3f\n", stepFactor); return new ElmBalancer(m, stepFactor, verbosity); } diff --git a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc index 085b95804..e3b0f6918 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc @@ -141,16 +141,16 @@ namespace { APF_ITERATE(PeerEntSet, peerVerts, pv) { const int dest = pv->first; double vw = weight(peerVerts[dest]); - PCU_COMM_PACK(dest, vw); + mesh->getPCU()->Pack(dest, vw); double ew = weight(peerEdges[dest]); - PCU_COMM_PACK(dest, ew); + mesh->getPCU()->Pack(dest, ew); } mesh->getPCU()->Send(); MigrComm incoming; double vw, ew; while (mesh->getPCU()->Listen()) { - PCU_COMM_UNPACK(vw); - PCU_COMM_UNPACK(ew); + mesh->getPCU()->Unpack(vw); + mesh->getPCU()->Unpack(ew); incoming.insert(Migr(mesh->getPCU()->Sender(),vw,ew)); //Migr ctor does not exist } @@ -184,15 +184,15 @@ namespace { mesh->getPCU()->Begin(); APF_ITERATE(Midd, accept, acc) { - PCU_COMM_PACK(acc->first, acc->second.a); - PCU_COMM_PACK(acc->first, acc->second.b); + mesh->getPCU()->Pack(acc->first, acc->second.a); + mesh->getPCU()->Pack(acc->first, acc->second.b); } mesh->getPCU()->Send(); Midd* capacity = new Midd; dtwo outw; while (mesh->getPCU()->Listen()) { - PCU_COMM_UNPACK(outw.a); - PCU_COMM_UNPACK(outw.b); + mesh->getPCU()->Unpack(outw.a); + mesh->getPCU()->Unpack(outw.b); (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; diff --git a/parma/diffMC/parma_entWeights.cc b/parma/diffMC/parma_entWeights.cc index b7305cc88..0a10201ce 100644 --- a/parma/diffMC/parma_entWeights.cc +++ b/parma/diffMC/parma_entWeights.cc @@ -6,12 +6,12 @@ namespace parma { double getMaxWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return PCU_Max_Double(locW); + return m->getPCU()->Max(locW); } double getAvgWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return PCU_Add_Double(locW) / m->getPCU()->Peers(); + return m->getPCU()->Add(locW) / m->getPCU()->Peers(); } double getWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { @@ -68,12 +68,12 @@ namespace parma { const Sides::Item* side; s->begin(); while( (side = s->iterate()) ) - PCU_COMM_PACK(side->first, weight); + m->getPCU()->Pack(side->first, weight); s->end(); m->getPCU()->Send(); while (m->getPCU()->Listen()) { double otherWeight; - PCU_COMM_UNPACK(otherWeight); + m->getPCU()->Unpack(otherWeight); set(m->getPCU()->Sender(), otherWeight); } } diff --git a/parma/diffMC/parma_ghostWeights.cc b/parma/diffMC/parma_ghostWeights.cc index 96c1b1efa..d37e587b8 100644 --- a/parma/diffMC/parma_ghostWeights.cc +++ b/parma/diffMC/parma_ghostWeights.cc @@ -80,7 +80,7 @@ namespace { current=next; next.clear(); } - PCU_Debug_Print("ghostW peer %d vtx %f edge %f elm %f\n", + m->getPCU()->DebugPrint("ghostW peer %d vtx %f edge %f elm %f\n", peer, weight[0], weight[1], weight[elmDim]); return weight; } @@ -227,7 +227,7 @@ namespace parma { findGhosts(finder, s); exchangeGhostsFrom(); exchange(); - PCU_Debug_Print("totW vtx %f edge %f elm %f\n", + m->getPCU()->DebugPrint("totW vtx %f edge %f elm %f\n", weight[0], weight[1], weight[dim]); } ~GhostWeights() { diff --git a/parma/diffMC/parma_graphDist.cc b/parma/diffMC/parma_graphDist.cc index 89f2db853..804dfff59 100644 --- a/parma/diffMC/parma_graphDist.cc +++ b/parma/diffMC/parma_graphDist.cc @@ -63,7 +63,7 @@ namespace { rsum[i] = rsum[i-1] + rmax[i-1] + 1 + maxDistanceIncrease; for(unsigned i=1; igetPCU()->DebugPrint("offset %u is %u\n", i, rsum[i]); // Go backwards so that the largest bdry vtx changes are made first // and won't be augmented in subsequent bdry traversals. @@ -215,7 +215,7 @@ namespace { }; apf::MeshTag* updateDistance(apf::Mesh* m) { - PCU_Debug_Print("updateDistance\n"); + m->getPCU()->DebugPrint("updateDistance\n"); apf::MeshTag* dist = parma::getDistTag(m); parma::DistanceQueue pq(m); getBdryVtx(m,dist,pq); @@ -267,7 +267,7 @@ namespace parma_ordering { if( ! c->has(e) ) continue; cnt++; int d; m->getIntTag(e,dt,&d); - PCU_Debug_Print("cnt %d d %d hasTag %d\n", cnt, d, m->hasTag(e,order)); + m->getPCU()->DebugPrint("cnt %d d %d hasTag %d\n", cnt, d, m->hasTag(e,order)); if( !m->hasTag(e,order) && d > rmax ) { rmax = d; emax = e; @@ -284,7 +284,7 @@ namespace parma_ordering { for(int i=TO_INT(c.size())-1; i>=0; i--) { CompContains* contains = new CompContains(c,i); apf::MeshEntity* src = getMaxDistSeed(m,contains,dist,order); - PCU_Debug_Print("comp %d starting vertex found? %d\n", i, (src != NULL)); + m->getPCU()->DebugPrint("comp %d starting vertex found? %d\n", i, (src != NULL)); start = bfs(m, contains, src, order, start); PCU_ALWAYS_ASSERT(check == c.getIdChecksum()); delete contains; @@ -342,12 +342,12 @@ namespace parma_ordering { la += abs(vid-uid); } m->end(it); - PCU_Debug_Print("la %d\n", la); - long tot=PCU_Add_Long(TO_LONG(la)); - int max=PCU_Max_Int(la); - int min=PCU_Min_Int(la); + m->getPCU()->DebugPrint("la %d\n", la); + long tot=m->getPCU()->Add(TO_LONG(la)); + int max=m->getPCU()->Max(la); + int min=m->getPCU()->Min(la); double avg = TO_DOUBLE(tot)/m->getPCU()->Peers(); - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) parmaCommons::status("la min %d max %d avg %.3f\n", min, max, avg); PCU_ALWAYS_ASSERT(check == m->getTagChecksum(order,apf::Mesh::VERTEX)); if( setOrder ) @@ -365,7 +365,7 @@ namespace parma { if( hasDistance(m) ) { t = updateDistance(m); } else { - PCU_Debug_Print("computeDistance\n"); + m->getPCU()->DebugPrint("computeDistance\n"); dcComponents c = dcComponents(m); t = computeDistance(m,c); if( m->getPCU()->Peers() > 1 && !c.numIso() ) @@ -384,7 +384,7 @@ namespace parma { } apf::MeshTag* Parma_BfsReorder(apf::Mesh* m, int) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); PCU_ALWAYS_ASSERT( !hasDistance(m) ); parma::dcComponents c = parma::dcComponents(m); const unsigned checkIds = c.getIdChecksum(); @@ -403,6 +403,6 @@ apf::MeshTag* Parma_BfsReorder(apf::Mesh* m, int) { PCU_ALWAYS_ASSERT(checkIds == c.getIdChecksum()); PCU_ALWAYS_ASSERT(check == m->getTagChecksum(dist,apf::Mesh::VERTEX)); m->destroyTag(dist); - parmaCommons::printElapsedTime(__func__,PCU_Time()-t0); + parmaCommons::printElapsedTime(__func__,pcu::Time()-t0); return order; } diff --git a/parma/diffMC/parma_ltSelector.cc b/parma/diffMC/parma_ltSelector.cc index bb15cb458..ebc3f91bd 100644 --- a/parma/diffMC/parma_ltSelector.cc +++ b/parma/diffMC/parma_ltSelector.cc @@ -111,13 +111,13 @@ namespace { mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, sendingEnts, s) - PCU_COMM_PACK(s->first, s->second); + mesh->getPCU()->Pack(s->first, s->second); mesh->getPCU()->Send(); MigrComm incoming; //map double w; while (mesh->getPCU()->Listen()) { - PCU_COMM_UNPACK(w); + mesh->getPCU()->Unpack(w); incoming.insert(Migr(mesh->getPCU()->Sender(),w)); } @@ -138,12 +138,12 @@ namespace { mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, accept, a) - PCU_COMM_PACK(a->first, a->second); + mesh->getPCU()->Pack(a->first, a->second); mesh->getPCU()->Send(); parma::Mid* capacity = new parma::Mid; double outw; while (mesh->getPCU()->Listen()) { - PCU_COMM_UNPACK(outw); + mesh->getPCU()->Unpack(outw); (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; diff --git a/parma/diffMC/parma_step.cc b/parma/diffMC/parma_step.cc index f23b7943e..846a6c8ee 100644 --- a/parma/diffMC/parma_step.cc +++ b/parma/diffMC/parma_step.cc @@ -35,11 +35,11 @@ namespace parma { if ( stop->stop(imb,maxImb) ) return false; apf::Migration* plan = selects->run(targets); - int planSz = PCU_Add_Int(plan->count()); - const double t0 = PCU_Time(); + int planSz = m->getPCU()->Add(plan->count()); + const double t0 = pcu::Time(); m->migrate(plan); if ( !m->getPCU()->Self() && verbosity ) - status("%d elements migrated in %f seconds\n", planSz, PCU_Time()-t0); + status("%d elements migrated in %f seconds\n", planSz, pcu::Time()-t0); if( verbosity > 1 ) Parma_PrintPtnStats(m, "endStep", (verbosity>2)); return true; diff --git a/parma/diffMC/parma_vtxBalancer.cc b/parma/diffMC/parma_vtxBalancer.cc index 510681562..0f5ac0aa8 100644 --- a/parma/diffMC/parma_vtxBalancer.cc +++ b/parma/diffMC/parma_vtxBalancer.cc @@ -51,7 +51,7 @@ namespace { apf::Balancer* Parma_MakeVtxBalancer(apf::Mesh* m, double stepFactor, int verbosity) { - if( !PCU_Comm_Self() && verbosity ) + if( !m->getPCU()->Self() && verbosity ) status("stepFactor %.3f\n", stepFactor); return new VtxBalancer(m, stepFactor, verbosity); } diff --git a/parma/diffMC/parma_vtxPtnWriter.cc b/parma/diffMC/parma_vtxPtnWriter.cc index d6d03f850..0c6e0f091 100644 --- a/parma/diffMC/parma_vtxPtnWriter.cc +++ b/parma/diffMC/parma_vtxPtnWriter.cc @@ -21,7 +21,7 @@ namespace { } public: Ptn(apf::Mesh* m) { - const long totv = PCU_Add_Long(countOwned(m)); + const long totv = m->getPCU()->Add(countOwned(m)); c = pp = totv / m->getPCU()->Peers(); f = pp * m->getPCU()->Self(); const int remainder = totv % m->getPCU()->Peers(); @@ -52,14 +52,14 @@ namespace { while( (vtx = m->iterate(itr)) ) { if( parma::isOwned(m, vtx) ) { m->getIntTag(vtx, t, &id); - PCU_COMM_PACK(p.getWriter(id), id); + m->getPCU()->Pack(p.getWriter(id), id); } } m->end(itr); m->getPCU()->Send(); while( m->getPCU()->Receive() ) { int id = 0; - PCU_COMM_UNPACK(id); + m->getPCU()->Unpack(id); const int idx = id - p.first(); PCU_ALWAYS_ASSERT(idx >= 0 && idx < p.count()); ptn[idx] = m->getPCU()->Sender(); diff --git a/parma/diffMC/parma_vtxSelector.cc b/parma/diffMC/parma_vtxSelector.cc index 587012990..aed10ec4b 100644 --- a/parma/diffMC/parma_vtxSelector.cc +++ b/parma/diffMC/parma_vtxSelector.cc @@ -158,7 +158,7 @@ namespace parma { } destroy(peers); } - PCU_Debug_Print("sent %u disconnected cavities\n", dcCnt); + mesh->getPCU()->DebugPrint("sent %u disconnected cavities\n", dcCnt); delete bdryVerts; return planW; } diff --git a/parma/parma.cc b/parma/parma.cc index f3a64cd57..1e0d5c10d 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -168,8 +168,8 @@ void Parma_GetEntImbalance(apf::Mesh* mesh, double (*entImb)[4]) { dims = TO_SIZET(mesh->getDimension()) + 1; for(size_t i=0; i < dims; i++) tot[i] = (*entImb)[i] = mesh->count(TO_INT(i)); - PCU_Add_Doubles(tot, dims); - PCU_Max_Doubles(*entImb, dims); + mesh->getPCU()->Add(tot, dims); + mesh->getPCU()->Max(*entImb, dims); for(size_t i=0; i < dims; i++) (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) @@ -183,8 +183,8 @@ void Parma_GetWeightedEntImbalance(apf::Mesh* mesh, apf::MeshTag* w, double tot[4] = {0,0,0,0}; for(size_t i=0; i < dims; i++) tot[i] = (*entImb)[i]; - PCU_Add_Doubles(tot, TO_SIZET(dims)); - PCU_Max_Doubles(*entImb, TO_SIZET(dims)); + mesh->getPCU()->Add(tot, TO_SIZET(dims)); + mesh->getPCU()->Max(*entImb, TO_SIZET(dims)); for(size_t i=0; i < dims; i++) (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) @@ -200,8 +200,8 @@ double Parma_GetWeightedEntImbalance(apf::Mesh* m, apf::MeshTag* w, while ((e = m->iterate(it))) sum += getEntWeight(m, e, w); m->end(it); - double tot = PCU_Add_Double(sum); - double max = PCU_Max_Double(sum); + double tot = m->getPCU()->Add(sum); + double max = m->getPCU()->Max(sum); return max/(tot/m->getPCU()->Peers()); } @@ -210,9 +210,9 @@ void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, mii nborToShared; getNeighborCounts(m,nborToShared); loc = TO_INT(nborToShared.size())-1; - max = PCU_Max_Int(loc); - avg = TO_DOUBLE(PCU_Add_Int(loc)) / m->getPCU()->Peers(); - numMaxParts = PCU_Add_Int( (loc==max) ); + max = m->getPCU()->Max(loc); + avg = TO_DOUBLE(m->getPCU()->Add(loc)) / m->getPCU()->Peers(); + numMaxParts = m->getPCU()->Add( (int)(loc==max) ); } void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { @@ -224,7 +224,7 @@ void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { for(int i=0; isecond == i+1 ) smallCnt[i]++; - PCU_Add_Ints(smallCnt,small); + m->getPCU()->Add(smallCnt,small); if( !m->getPCU()->Self() ) { std::stringstream ss; for(int i=0; igetPCU()->Max(loc); int smallest = INT_MAX; if( loc == max ) { APF_ITERATE(mii, nborToShared, nbor) if( nbor->second < smallest ) smallest = nbor->second; } - return PCU_Min_Int(smallest); + return m->getPCU()->Min(smallest); } void Parma_GetOwnedBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, @@ -271,8 +271,8 @@ void Parma_GetMdlBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, void Parma_GetDisconnectedStats(apf::Mesh* m, int& max, double& avg, int& loc) { dcPart dc(m); loc = TO_INT(dc.getNumDcComps()); - max = PCU_Max_Int(loc); - avg = TO_DOUBLE( PCU_Add_Int(loc) ) / m->getPCU()->Peers(); + max = m->getPCU()->Max(loc); + avg = TO_DOUBLE( m->getPCU()->Add(loc) ) / m->getPCU()->Peers(); } void Parma_ProcessDisconnectedParts(apf::Mesh* m) { @@ -297,24 +297,24 @@ void Parma_PrintPtnStats(apf::Mesh* m, std::string key, bool fine) { } void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, bool fine) { - PCU_Debug_Print("%s vtx %lu\n", key.c_str(), m->count(0)); - PCU_Debug_Print("%s edge %lu\n", key.c_str(), m->count(1)); - PCU_Debug_Print("%s face %lu\n", key.c_str(), m->count(2)); + m->getPCU()->DebugPrint("%s vtx %lu\n", key.c_str(), m->count(0)); + m->getPCU()->DebugPrint("%s edge %lu\n", key.c_str(), m->count(1)); + m->getPCU()->DebugPrint("%s face %lu\n", key.c_str(), m->count(2)); if( m->getDimension() == 3 ) - PCU_Debug_Print("%s rgn %lu\n", key.c_str(), m->count(3)); + m->getPCU()->DebugPrint("%s rgn %lu\n", key.c_str(), m->count(3)); int maxDc = 0; double avgDc = 0; int locDc = 0; Parma_GetDisconnectedStats(m, maxDc, avgDc, locDc); - PCU_Debug_Print("%s dc %d\n", key.c_str(), locDc); + m->getPCU()->DebugPrint("%s dc %d\n", key.c_str(), locDc); int maxNb = 0, maxNbParts = 0; double avgNb = 0; int locNb = 0; Parma_GetNeighborStats(m, maxNb, maxNbParts, avgNb, locNb); int smallSideMaxNbPart = Parma_GetSmallestSideMaxNeighborParts(m); - PCU_Debug_Print("%s neighbors %d\n", key.c_str(), locNb); + m->getPCU()->DebugPrint("%s neighbors %d\n", key.c_str(), locNb); int locV[3], minV[3], maxV[3]; long totV[3]; @@ -322,20 +322,20 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, Parma_GetOwnedBdryVtxStats(m, locV[0], totV[0], minV[0], maxV[0], avgV[0]); Parma_GetSharedBdryVtxStats(m, locV[1], totV[1], minV[1], maxV[1], avgV[1]); Parma_GetMdlBdryVtxStats(m, locV[2], totV[2], minV[2], maxV[2], avgV[2]); - PCU_Debug_Print("%s ownedBdryVtx %d\n", key.c_str(), locV[0]); - PCU_Debug_Print("%s sharedBdryVtx %d\n", key.c_str(), locV[1]); - PCU_Debug_Print("%s mdlBdryVtx %d\n", key.c_str(), locV[2]); + m->getPCU()->DebugPrint("%s ownedBdryVtx %d\n", key.c_str(), locV[0]); + m->getPCU()->DebugPrint("%s sharedBdryVtx %d\n", key.c_str(), locV[1]); + m->getPCU()->DebugPrint("%s mdlBdryVtx %d\n", key.c_str(), locV[2]); int surf = numSharedSides(m); double vol = TO_DOUBLE( m->count(m->getDimension()) ); double surfToVol = surf/vol; - double minSurfToVol = PCU_Min_Double(surfToVol); - double maxSurfToVol = PCU_Max_Double(surfToVol); - double avgSurfToVol = PCU_Add_Double(surfToVol) / m->getPCU()->Peers(); - PCU_Debug_Print("%s sharedSidesToElements %.3f\n", key.c_str(), surfToVol); + double minSurfToVol = m->getPCU()->Min(surfToVol); + double maxSurfToVol = m->getPCU()->Max(surfToVol); + double avgSurfToVol = m->getPCU()->Add(surfToVol) / m->getPCU()->Peers(); + m->getPCU()->DebugPrint("%s sharedSidesToElements %.3f\n", key.c_str(), surfToVol); int empty = (m->count(m->getDimension()) == 0 ) ? 1 : 0; - empty = PCU_Add_Int(empty); + empty = m->getPCU()->Add(empty); double imb[4] = {0, 0, 0, 0}; Parma_GetWeightedEntImbalance(m,w,&imb); @@ -343,12 +343,12 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, if (fine) writeFineStats(m, key, locDc, locNb, locV, surf, vol); - PCU_Debug_Print("%s vtxAdjacentNeighbors ", key.c_str()); + m->getPCU()->DebugPrint("%s vtxAdjacentNeighbors ", key.c_str()); apf::Parts peers; apf::getPeers(m,0,peers); APF_ITERATE(apf::Parts,peers,p) - PCU_Debug_Print("%d ", *p); - PCU_Debug_Print("\n"); + m->getPCU()->DebugPrint("%d ", *p); + m->getPCU()->DebugPrint("\n"); if( 0 == m->getPCU()->Self() ) { status("%s disconnected %d %.3f\n", @@ -420,7 +420,7 @@ int Parma_MisNumbering(apf::Mesh* m, int d) { } iter++; misSize = (misNumber != -1); - misSize = PCU_Add_Int(misSize); + misSize = m->getPCU()->Add(misSize); } return misNumber; } diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index 23a608ffd..5db3d081c 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -32,7 +32,7 @@ class PCU { template int Pack(int to_rank, T& data) noexcept { return Pack(to_rank, &(data), sizeof(data)); } - template int Pack(int to_rank, T* data) noexcept { + template int Pack(int to_rank, T*& data) noexcept { return Pack(to_rank, &(data), sizeof(data)); } @@ -45,7 +45,7 @@ class PCU { template int Unpack(T& data) noexcept { return Unpack(&(data), sizeof(data)); } - template int Unpack(T* data) noexcept { + template int Unpack(T*& data) noexcept { return Unpack(&(data), sizeof(data)); } /*IPComMan replacement API*/ From 33a66217e193aeb5970c3ba5a054a6257f6ed8fd Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 5 Feb 2024 14:31:54 -0500 Subject: [PATCH 046/141] Making new versions of functions to handle PCUHandle --- mds/mds_apf.c | 65 ++++++++++++++++++++++++++++++----------------- mds/mds_apf.h | 9 +++++++ mds/mds_smb.c | 62 +++++++++++++++++++++++++++----------------- pcu/PCU2.h | 6 ++--- pcu/PCUObj.cc | 4 ++- pcu/PCUObj.h | 1 + pcu/pcu.cc | 6 +++++ pcu/pcu2.cc | 2 +- pcu/pcu_defines.h | 14 +++++++++- pcu/pcu_io.c | 34 +++++++++++++++++++------ pcu/pcu_io.h | 4 +++ 11 files changed, 146 insertions(+), 61 deletions(-) diff --git a/mds/mds_apf.c b/mds/mds_apf.c index 262cd2542..d303dd47b 100644 --- a/mds/mds_apf.c +++ b/mds/mds_apf.c @@ -11,7 +11,8 @@ #include "mds_apf.h" #include #include -#include +//#include +#include struct mds_apf* mds_apf_create(struct gmi_model* model, int d, mds_id cap[MDS_TYPES]) @@ -148,23 +149,23 @@ int mds_model_id(struct mds_apf* m, struct gmi_ent* model) return gmi_tag(m->user_model, model); } -static void downs_to_copy(struct mds_set* s, +static void downs_to_copy(PCUHandle h, struct mds_set* s, struct mds_copy c) { int i; - PCU_COMM_PACK(c.p, c.e); + PCU_COMM_PACK2(h, c.p, c.e); for (i = 0; i < s->n; ++i) - PCU_COMM_PACK(c.p, s->e[i]); + PCU_COMM_PACK2(h, c.p, s->e[i]); } static void downs_to_copies( - struct mds* m, mds_id e, struct mds_copies* c) + PCUHandle h, struct mds* m, mds_id e, struct mds_copies* c) { int i; struct mds_set s; mds_get_adjacent(m, e, mds_dim[mds_type(e)] - 1, &s); for (i = 0; i < c->n; ++i) - downs_to_copy(&s, c->c[i]); + downs_to_copy(h, &s, c->c[i]); } static void change_down(struct mds* m, mds_id e, struct mds_set* s) @@ -219,19 +220,19 @@ static void rotate_set(struct mds_set* in, int r, struct mds_set* out) out->e[i] = in->e[(i + r) % in->n]; } -static int recv_down_copies(struct mds_net* net, struct mds* m) +static int recv_down_copies(PCUHandle h, struct mds_net* net, struct mds* m) { mds_id e; struct mds_set s; struct mds_set rs; struct mds_set s2; int i; - int from = PCU_Comm_Sender(); - PCU_COMM_UNPACK(e); + int from = PCU_Comm_Sender2(h); + PCU_COMM_UNPACK2(h, e); mds_get_adjacent(m, e, mds_dim[mds_type(e)] - 1, &s); rs.n = s.n; for (i = 0; i < s.n; ++i) - PCU_COMM_UNPACK(rs.e[i]); + PCU_COMM_UNPACK2(h, rs.e[i]); if (compare_copy_sets(net, &s, from, &rs)) return 0; for (i = -s.n; i < s.n; ++i) { @@ -251,55 +252,73 @@ static int copy_less(struct mds_copy a, struct mds_copy b) return a.e < b.e; } -static int owns_copies(mds_id e, struct mds_copies* c) +static int owns_copies(PCUHandle h, mds_id e, struct mds_copies* c) { int i; struct mds_copy mc; mc.e = e; - mc.p = PCU_Comm_Self(); + mc.p = PCU_Comm_Self2(h); for (i = 0; i < c->n; ++i) if (copy_less(c->c[i], mc)) return 0; return 1; } -static int align_copies(struct mds_net* net, struct mds* m) +static int align_copies(PCUHandle h, struct mds_net* net, struct mds* m) { int d; mds_id e; struct mds_copies* c; int did_change = 0; for (d = 1; d < m->d; ++d){ - PCU_Comm_Begin(); + PCU_Comm_Begin2(h); for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) { c = mds_get_copies(net, e); if (!c) continue; - if (owns_copies(e, c)) - downs_to_copies(m, e, c); + if (owns_copies(h, e, c)) + downs_to_copies(h, m, e, c); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) - if (recv_down_copies(net, m)) + PCU_Comm_Send2(h); + while (PCU_Comm_Receive2(h)) + if (recv_down_copies(h, net, m)) did_change = 1; } - return PCU_Or(did_change); + return PCU_Or2(h, did_change); } int mds_align_matches(struct mds_apf* m) { - return align_copies(&m->matches, &m->mds); + PCUHandle h = PCU_Get_Global_Handle(); + return mds_align_matches2(h, m); +} + +int mds_align_matches2(PCUHandle h, struct mds_apf* m) +{ + return align_copies(h, &m->matches, &m->mds); } // seol int mds_align_ghosts(struct mds_apf* m) { - return align_copies(&m->ghosts, &m->mds); + PCUHandle h = PCU_Get_Global_Handle(); + return mds_align_ghosts2(h, m); +} + +int mds_align_ghosts2(PCUHandle h, struct mds_apf* m) +{ + return align_copies(h, &m->ghosts, &m->mds); } int mds_align_remotes(struct mds_apf* m) { - return align_copies(&m->remotes, &m->mds); + PCUHandle h = PCU_Get_Global_Handle(); + return mds_align_remotes2(h, m); +} + +int mds_align_remotes2(PCUHandle h, struct mds_apf* m) +{ + return align_copies(h, &m->remotes, &m->mds); } void mds_update_model_for_entity(struct mds_apf* m, mds_id e, diff --git a/mds/mds_apf.h b/mds/mds_apf.h index fb51e2ff2..692024fdf 100644 --- a/mds/mds_apf.h +++ b/mds/mds_apf.h @@ -26,6 +26,8 @@ struct pcu_file; struct gmi_model; struct gmi_ent; +typedef struct PCUHandle PCUHandle; + struct mds_apf { struct mds mds; struct mds_tags tags; @@ -64,15 +66,22 @@ int mds_model_id(struct mds_apf* m, struct gmi_ent* model); struct mds_apf* mds_read_smb(struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh); +struct mds_apf* mds_read_smb2(PCUHandle h, struct gmi_model* model, const char* pathname, + int ignore_peers, void* apf_mesh); struct mds_apf* mds_write_smb(struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh); +struct mds_apf* mds_write_smb2(PCUHandle h, struct mds_apf* m, const char* pathname, + int ignore_peers, void* apf_mesh); void mds_verify(struct mds_apf* m); void mds_verify_residence(struct mds_apf* m, mds_id e); int mds_align_matches(struct mds_apf* m); +int mds_align_matches2(PCUHandle h, struct mds_apf* m); int mds_align_ghosts(struct mds_apf* m); +int mds_align_ghosts2(PCUHandle h, struct mds_apf* m); int mds_align_remotes(struct mds_apf* m); +int mds_align_remotes2(PCUHandle h, struct mds_apf* m); void mds_derive_model(struct mds_apf* m); void mds_update_model_for_entity(struct mds_apf* m, mds_id e, diff --git a/mds/mds_smb.c b/mds/mds_smb.c index f0590d679..d91ae751b 100644 --- a/mds/mds_smb.c +++ b/mds/mds_smb.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -51,6 +51,8 @@ enum { #define MAX_PEERS (10*1000) #define MAX_TAGS (100) +typedef struct PCUHandle PCUHandle; + static int smb2mds(int smb_type) { int const table[SMB_TYPES] = @@ -116,7 +118,7 @@ static void write_links(struct pcu_file* f, struct mds_links* l) pcu_write_unsigneds(f, l->l[i], l->n[i]); } -static void read_header(struct pcu_file* f, unsigned* version, unsigned* dim, +static void read_header(PCUHandle h, struct pcu_file* f, unsigned* version, unsigned* dim, int ignore_peers) { unsigned magic, np; @@ -126,12 +128,12 @@ static void read_header(struct pcu_file* f, unsigned* version, unsigned* dim, PCU_READ_UNSIGNED(f, *dim); PCU_READ_UNSIGNED(f, np); if (*version >= 1 && (!ignore_peers)) - if (np != (unsigned)PCU_Comm_Peers()) + if (np != (unsigned)PCU_Comm_Peers2(h)) reel_fail("To whom it may concern\n" "the # of mesh partitions != the # of MPI ranks"); } -static void write_header(struct pcu_file* f, unsigned dim, +static void write_header(PCUHandle h, struct pcu_file* f, unsigned dim, int ignore_peers) { unsigned magic = 0; @@ -143,7 +145,7 @@ static void write_header(struct pcu_file* f, unsigned dim, if (ignore_peers) np = 1; else - np = (unsigned)PCU_Comm_Peers(); + np = (unsigned)PCU_Comm_Peers2(h); PCU_WRITE_UNSIGNED(f, np); } @@ -557,7 +559,7 @@ static void write_matches(struct pcu_file* f, struct mds_apf* m, write_type_matches(f, m, smb2mds(t), ignore_peers); } -static struct mds_apf* read_smb(struct gmi_model* model, const char* filename, +static struct mds_apf* read_smb(PCUHandle h, struct gmi_model* model, const char* filename, int zip, int ignore_peers, void* apf_mesh) { struct mds_apf* m; @@ -569,9 +571,9 @@ static struct mds_apf* read_smb(struct gmi_model* model, const char* filename, int i; unsigned tmp; unsigned pi, pj; - f = pcu_fopen(filename, 0, zip); + f = pcu_fopen2(h, filename, 0, zip); PCU_ALWAYS_ASSERT(f); - read_header(f, &version, &dim, ignore_peers); + read_header(h, f, &version, &dim, ignore_peers); pcu_read_unsigneds(f, n, SMB_TYPES); for (i = 0; i < MDS_TYPES; ++i) { tmp = n[mds2smb(i)]; @@ -612,15 +614,15 @@ static void write_coords(struct pcu_file* f, struct mds_apf* m) pcu_write_doubles(f, &m->param[0][0], count); } -static void write_smb(struct mds_apf* m, const char* filename, +static void write_smb(PCUHandle h, struct mds_apf* m, const char* filename, int zip, int ignore_peers, void* apf_mesh) { struct pcu_file* f; unsigned n[SMB_TYPES] = {0}; int i; - f = pcu_fopen(filename, 1, zip); + f = pcu_fopen2(h, filename, 1, zip); PCU_ALWAYS_ASSERT(f); - write_header(f, m->mds.d, ignore_peers); + write_header(h, f, m->mds.d, ignore_peers); for (i = 0; i < MDS_TYPES; ++i) n[mds2smb(i)] = m->mds.end[i]; pcu_write_unsigneds(f, n, SMB_TYPES); @@ -686,7 +688,7 @@ static void safe_mkdir(const char* path, mode_t mode) reel_fail("MDS: could not create directory \"%s\"\n", path); } -static char* handle_path(const char* in, int is_write, int* zip, +static char* handle_path(PCUHandle h, const char* in, int is_write, int* zip, int ignore_peers) { static const char* zippre = "bz2:"; @@ -694,7 +696,7 @@ static char* handle_path(const char* in, int is_write, int* zip, size_t bufsize; char* path; mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - int self = PCU_Comm_Self(); + int self = PCU_Comm_Self2(h); bufsize = strlen(in) + 256; path = malloc(bufsize); strcpy(path, in); @@ -710,14 +712,14 @@ static char* handle_path(const char* in, int is_write, int* zip, if (is_write) { if (!self) safe_mkdir(path, dir_perm); - PCU_Barrier(); + PCU_Barrier2(h); } - if (PCU_Comm_Peers() > SMB_FANOUT) { + if (PCU_Comm_Peers2(h) > SMB_FANOUT) { append(path, bufsize, "%d/", self / SMB_FANOUT); if (is_write) { if (self % SMB_FANOUT == 0) safe_mkdir(path, dir_perm); - PCU_Barrier(); + PCU_Barrier2(h); } } } else if (ends_with(path, smbext)) { @@ -731,12 +733,19 @@ static char* handle_path(const char* in, int is_write, int* zip, struct mds_apf* mds_read_smb(struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_read_smb2(h, model, pathname, ignore_peers, apf_mesh); +} + +struct mds_apf* mds_read_smb2(PCUHandle h, struct gmi_model* model, const char* pathname, + int ignore_peers, void* apf_mesh) { char* filename; int zip; struct mds_apf* m; - filename = handle_path(pathname, 0, &zip, ignore_peers); - m = read_smb(model, filename, zip, ignore_peers, apf_mesh); + filename = handle_path(h, pathname, 0, &zip, ignore_peers); + m = read_smb(h, model, filename, zip, ignore_peers, apf_mesh); free(filename); return m; } @@ -752,20 +761,27 @@ static int is_compact(struct mds_apf* m) struct mds_apf* mds_write_smb(struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_write_smb2(h, m, pathname, ignore_peers, apf_mesh); +} + +struct mds_apf* mds_write_smb2(PCUHandle h, struct mds_apf* m, const char* pathname, + int ignore_peers, void* apf_mesh) { const char* reorderWarning ="MDS: reordering before writing smb files\n"; char* filename; int zip; if (ignore_peers && (!is_compact(m))) { - if(!PCU_Comm_Self()) lion_eprint(1, "%s", reorderWarning); + if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); m = mds_reorder(m, 1, mds_number_verts_bfs(m)); } - if ((!ignore_peers) && PCU_Or(!is_compact(m))) { - if(!PCU_Comm_Self()) lion_eprint(1, "%s", reorderWarning); + if ((!ignore_peers) && PCU_Or2(h, !is_compact(m))) { + if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); m = mds_reorder(m, 0, mds_number_verts_bfs(m)); } - filename = handle_path(pathname, 1, &zip, ignore_peers); - write_smb(m, filename, zip, ignore_peers, apf_mesh); + filename = handle_path(h, pathname, 1, &zip, ignore_peers); + write_smb(h, m, filename, zip, ignore_peers, apf_mesh); free(filename); return m; } diff --git a/pcu/PCU2.h b/pcu/PCU2.h index c9da30ddf..914e25299 100644 --- a/pcu/PCU2.h +++ b/pcu/PCU2.h @@ -13,9 +13,7 @@ extern "C" { #include #endif -struct PCUHandle { - void* ptr; -}; +typedef struct PCUHandle PCUHandle; int PCU_Comm_Init2(PCUHandle* h); int PCU_Comm_Free2(PCUHandle* h); @@ -112,7 +110,7 @@ double PCU_Time2(void); double PCU_GetMem2(void); /*Access global variable*/ - +PCUHandle PCU_Get_Global_Handle(void); diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index 0a92f5c6f..0e572d883 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -147,10 +147,12 @@ void PCU::DebugOpen() noexcept { append(path, bufsize, "%s", "debug"); if (!msg_->file) - msg_->file = pcu_open_parallel(path, "txt"); + msg_->file = pcu_open_parallel2(GetCHandle(), path, "txt"); noto_free(path); } + + double GetMem() noexcept { return pcu_get_mem(); } void Protect() noexcept { reel_protect(); } double Time() noexcept { return MPI_Wtime(); } diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index 5db3d081c..2f81dbd2c 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -26,6 +26,7 @@ class PCU { [[nodiscard]] int Peers() const noexcept; [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; + [[nodiscard]] PCUHandle GetCHandle() {PCUHandle h; h.ptr=this; return h;} /*recommended message passing API*/ void Begin() noexcept; int Pack(int to_rank, const void *data, size_t size) noexcept; diff --git a/pcu/pcu.cc b/pcu/pcu.cc index bddff5c70..781d5219b 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -569,4 +569,10 @@ double PCU_GetMem(void) { return pcu::GetMem(); } */ pcu::PCU* PCU_GetGlobal(void) { return global_pcu; } +PCUHandle PCU_Get_Global_Handle(void) { + PCUHandle h; + h.ptr = static_cast(PCU_GetGlobal()); + return h; +} + } diff --git a/pcu/pcu2.cc b/pcu/pcu2.cc index 877ec0e97..593cc0bbf 100644 --- a/pcu/pcu2.cc +++ b/pcu/pcu2.cc @@ -503,7 +503,7 @@ double PCU_GetMem2(void) { return pcu::GetMem(); } /** \brief Return the global PCU */ -pcu::PCU* PCU_GetGlobal2(PCUHandle h) { return static_cast(h.ptr); } + diff --git a/pcu/pcu_defines.h b/pcu/pcu_defines.h index c39152d16..22957adab 100644 --- a/pcu/pcu_defines.h +++ b/pcu/pcu_defines.h @@ -10,4 +10,16 @@ #define PCU_FORMAT_ATTRIBUTE(format, ...) #endif -#endif // SCOREC_PCU_PCU_DEFINES_H +#ifdef __cplusplus +extern "C"{ +#endif + +struct PCUHandle { + void* ptr; +}; + +#ifdef __cplusplus +} +#endif + +#endif // SCOREC_PCU_PCU_DEFINES_H \ No newline at end of file diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index a89f366bd..ebdc20974 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,7 +8,8 @@ *******************************************************************************/ #include "pcu_io.h" -#include "PCU.h" +//#include "PCU.h" +#include "PCU2.h" #include "noto_malloc.h" #include "pcu_buffer.h" #include "pcu_util.h" @@ -159,17 +160,22 @@ static void close_compressed(pcu_file* pf) * removed. */ FILE* pcu_group_open(const char* path, bool write) { + PCUHandle h = PCU_Get_Global_Handle(); + return pcu_group_open2(h, path, write); +} + +FILE* pcu_group_open2(PCUHandle h, const char* path, bool write) { FILE* fp = NULL; - const int rank = PCU_Comm_Self(); + const int rank = PCU_Comm_Self2(h); const char* mode = write ? "w" : "r"; const int group_size = 4096; - const int q = PCU_Comm_Peers()/group_size; - const int r = PCU_Comm_Peers()%group_size; + const int q = PCU_Comm_Peers2(h)/group_size; + const int r = PCU_Comm_Peers2(h)%group_size; const int groups = q + ( r > 0 ); if(!rank && groups > 1) { fprintf(stderr, "pcu peers %d max group size %d posix groups %d\n", - PCU_Comm_Peers(), group_size, groups); + PCU_Comm_Peers2(h), group_size, groups); } for(int i=0; icompress = compress; pf->write = write; - pf->f = pcu_group_open(name, write); + pf->f = pcu_group_open2(h, name, write); if (!pf->f) { perror("pcu_fopen"); reel_fail("pcu_fopen couldn't open \"%s\"", name); @@ -356,13 +368,19 @@ void pcu_write_string (pcu_file * f, const char * p) } FILE* pcu_open_parallel(const char* prefix, const char* ext) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return pcu_open_parallel2(h, prefix, ext); +} + +FILE* pcu_open_parallel2(PCUHandle h, const char* prefix, const char* ext) { //max_rank_chars = strlen("4294967296"), 4294967296 = 2^32 ~= INT_MAX static const size_t max_rank_chars = 10; size_t path_size = strlen(prefix) + max_rank_chars + strlen(ext) + 1; char* path = noto_malloc(path_size); int rank; - PCU_Comm_Rank(&rank); + PCU_Comm_Rank2(h, &rank); snprintf(path,path_size,"%s%d.%s",prefix,rank,ext); FILE* file = fopen(path, "w"); noto_free(path); diff --git a/pcu/pcu_io.h b/pcu/pcu_io.h index 1ba0f48d7..ef7eaad0c 100644 --- a/pcu/pcu_io.h +++ b/pcu/pcu_io.h @@ -20,8 +20,10 @@ extern "C" { #endif struct pcu_file; +typedef struct PCUHandle PCUHandle; struct pcu_file* pcu_fopen(const char* path, bool write, bool compress); +struct pcu_file* pcu_fopen2(PCUHandle h, const char* path, bool write, bool compress); void pcu_fclose (struct pcu_file * pf); void pcu_read(struct pcu_file* f, char* p, size_t n); void pcu_write(struct pcu_file* f, const char* p, size_t n); @@ -35,7 +37,9 @@ void pcu_read_string(struct pcu_file* f, char** p); void pcu_write_string(struct pcu_file* f, const char* p); FILE* pcu_open_parallel(const char* prefix, const char* ext); +FILE* pcu_open_parallel2(PCUHandle h, const char* prefix, const char* ext); FILE* pcu_group_open(const char* path, bool write); +FILE* pcu_group_open2(PCUHandle h, const char* path, bool write); void pcu_swap_doubles(double* p, size_t n); void pcu_swap_unsigneds(unsigned* p, size_t n); From 8c5633a5a891d1d44baf8255b5b6f59559576da7 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 7 Feb 2024 10:33:14 -0500 Subject: [PATCH 047/141] phCook changes, changed phInput openfile_read to take PCUHandle, and it brings up issues --- phasta/phCook.cc | 16 ++++++++-------- phasta/phGeomBC.cc | 2 +- phasta/phInput.h | 3 ++- phasta/phOutput.h | 4 +++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 6c049f4e9..b38f99406 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -111,25 +111,25 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, }//end namespace namespace chef { - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(PCUHandle h, ph::Input&, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open(path, false);) + PHASTAIO_OPENTIME(f = pcu_group_open2(h, path, false);) return f; } - static FILE* openfile_write(ph::Output&, const char* path) { + static FILE* openfile_write(PCUHandle h, ph::Output&, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open(path, true);) + PHASTAIO_OPENTIME(f = pcu_group_open2(h, path, true);) return f; } - static FILE* openstream_write(ph::Output& out, const char* path) { + static FILE* openstream_write(PCUHandle h, ph::Output& out, const char* path) { FILE* f = NULL; PHASTAIO_OPENTIME(f = openGRStreamWrite(out.grs, path);) return f; } - static FILE* openstream_read(ph::Input& in, const char* path) { + static FILE* openstream_read(PCUHandle h, ph::Input& in, const char* path) { std::string fname(path); std::string restartStr("restart"); FILE* f = NULL; @@ -201,7 +201,7 @@ namespace ph { if ( in.writeRestartFiles ) { if(!m->getPCU()->Self()) lion_oprint(1,"write file-based restart file\n"); // store the value of the function pointer - FILE* (*fn)(Output& out, const char* path) = out.openfile_write; + FILE* (*fn)(PCUHandle h, Output& out, const char* path) = out.openfile_write; // set function pointer for file writing out.openfile_write = chef::openfile_write; ph::detachAndWriteSolution(in,out,m,subDirPath); //write restart @@ -216,7 +216,7 @@ namespace ph { if ( in.writeGeomBCFiles ) { if(!m->getPCU()->Self()) lion_oprint(1,"write additional geomBC file for visualization\n"); // store the value of the function pointer - FILE* (*fn)(Output& out, const char* path) = out.openfile_write; + FILE* (*fn)(PCUHandle h, Output& out, const char* path) = out.openfile_write; // set function pointer for file writing out.openfile_write = chef::openfile_write; ph::writeGeomBC(out, path, in.timeStepNumber); //write geombc for viz only diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc index 4f88d1f00..0e0a200bf 100644 --- a/phasta/phGeomBC.cc +++ b/phasta/phGeomBC.cc @@ -306,7 +306,7 @@ void writeGeomBC(Output& o, std::string path, int timestep) } path += buildGeomBCFileName(timestep_or_dat); phastaio_setfile(GEOMBC_WRITE); - FILE* f = o.openfile_write(o, path.c_str()); + FILE* f = o.openfile_write(m->getPCU()->GetCHandle(), o, path.c_str()); if (!f) { lion_eprint(1,"failed to open \"%s\"!\n", path.c_str()); abort(); diff --git a/phasta/phInput.h b/phasta/phInput.h index a6bf88c90..8dc304327 100644 --- a/phasta/phInput.h +++ b/phasta/phInput.h @@ -12,6 +12,7 @@ #include #include +#include "PCU2.h" struct RStream; @@ -158,7 +159,7 @@ class Input double meshqCrtn; double elementImbalance; double vertexImbalance; - FILE* (*openfile_read)(Input& in, const char* path); + FILE* (*openfile_read)(PCUHandle h, Input& in, const char* path); RStream* rs; /** \brief the flag for switch between simmetrix mesh and pumi-based mesh. avoid run incompatible APIs with simmetrix mesh */ diff --git a/phasta/phOutput.h b/phasta/phOutput.h index 6a93e9d96..74ee2c057 100644 --- a/phasta/phOutput.h +++ b/phasta/phOutput.h @@ -12,6 +12,8 @@ class MeshEntity; struct GRStream; +typedef struct PCUHandle PCUHandle; + namespace ph { struct EnsaArrays @@ -160,7 +162,7 @@ struct Output int nLayeredMeshVertices; /* number of layered mesh vertices */ bool hasDGInterface; int numRigidBody; - FILE* (*openfile_write)(Output& out, const char* path); + FILE* (*openfile_write)(PCUHandle h, Output& out, const char* path); GRStream* grs; AllBlocks blocks; EnsaArrays arrays; From eaeaa3deac5caa5d8e7c93ae78b7bdb2174c3fb6 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 8 Feb 2024 13:36:18 -0500 Subject: [PATCH 048/141] Reverting phasta changes --- phasta/phCook.cc | 19 +++++++++---------- phasta/phGeomBC.cc | 2 +- phasta/phInput.h | 5 +---- phasta/phOutput.h | 4 +--- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/phasta/phCook.cc b/phasta/phCook.cc index b38f99406..fe9636a8d 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -111,25 +111,25 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, }//end namespace namespace chef { - static FILE* openfile_read(PCUHandle h, ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open2(h, path, false);) + PHASTAIO_OPENTIME(f = pcu_group_open(path, false);) return f; } - static FILE* openfile_write(PCUHandle h, ph::Output&, const char* path) { + static FILE* openfile_write(ph::Output&, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open2(h, path, true);) + PHASTAIO_OPENTIME(f = pcu_group_open(path, true);) return f; } - static FILE* openstream_write(PCUHandle h, ph::Output& out, const char* path) { + static FILE* openstream_write(ph::Output& out, const char* path) { FILE* f = NULL; PHASTAIO_OPENTIME(f = openGRStreamWrite(out.grs, path);) return f; } - static FILE* openstream_read(PCUHandle h, ph::Input& in, const char* path) { + static FILE* openstream_read(ph::Input& in, const char* path) { std::string fname(path); std::string restartStr("restart"); FILE* f = NULL; @@ -201,7 +201,7 @@ namespace ph { if ( in.writeRestartFiles ) { if(!m->getPCU()->Self()) lion_oprint(1,"write file-based restart file\n"); // store the value of the function pointer - FILE* (*fn)(PCUHandle h, Output& out, const char* path) = out.openfile_write; + FILE* (*fn)(Output& out, const char* path) = out.openfile_write; // set function pointer for file writing out.openfile_write = chef::openfile_write; ph::detachAndWriteSolution(in,out,m,subDirPath); //write restart @@ -216,7 +216,7 @@ namespace ph { if ( in.writeGeomBCFiles ) { if(!m->getPCU()->Self()) lion_oprint(1,"write additional geomBC file for visualization\n"); // store the value of the function pointer - FILE* (*fn)(PCUHandle h, Output& out, const char* path) = out.openfile_write; + FILE* (*fn)(Output& out, const char* path) = out.openfile_write; // set function pointer for file writing out.openfile_write = chef::openfile_write; ph::writeGeomBC(out, path, in.timeStepNumber); //write geombc for viz only @@ -384,5 +384,4 @@ namespace chef { out.grs = grs; ph::preprocess(m,in,out); } -} - +} \ No newline at end of file diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc index 0e0a200bf..4f88d1f00 100644 --- a/phasta/phGeomBC.cc +++ b/phasta/phGeomBC.cc @@ -306,7 +306,7 @@ void writeGeomBC(Output& o, std::string path, int timestep) } path += buildGeomBCFileName(timestep_or_dat); phastaio_setfile(GEOMBC_WRITE); - FILE* f = o.openfile_write(m->getPCU()->GetCHandle(), o, path.c_str()); + FILE* f = o.openfile_write(o, path.c_str()); if (!f) { lion_eprint(1,"failed to open \"%s\"!\n", path.c_str()); abort(); diff --git a/phasta/phInput.h b/phasta/phInput.h index 8dc304327..bca7c842e 100644 --- a/phasta/phInput.h +++ b/phasta/phInput.h @@ -12,7 +12,6 @@ #include #include -#include "PCU2.h" struct RStream; @@ -159,7 +158,7 @@ class Input double meshqCrtn; double elementImbalance; double vertexImbalance; - FILE* (*openfile_read)(PCUHandle h, Input& in, const char* path); + FILE* (*openfile_read)(Input& in, const char* path); RStream* rs; /** \brief the flag for switch between simmetrix mesh and pumi-based mesh. avoid run incompatible APIs with simmetrix mesh */ @@ -217,5 +216,3 @@ int countScalarBCs(Input& in); } #endif - - diff --git a/phasta/phOutput.h b/phasta/phOutput.h index 74ee2c057..6a93e9d96 100644 --- a/phasta/phOutput.h +++ b/phasta/phOutput.h @@ -12,8 +12,6 @@ class MeshEntity; struct GRStream; -typedef struct PCUHandle PCUHandle; - namespace ph { struct EnsaArrays @@ -162,7 +160,7 @@ struct Output int nLayeredMeshVertices; /* number of layered mesh vertices */ bool hasDGInterface; int numRigidBody; - FILE* (*openfile_write)(PCUHandle h, Output& out, const char* path); + FILE* (*openfile_write)(Output& out, const char* path); GRStream* grs; AllBlocks blocks; EnsaArrays arrays; From 244a6d535455bbea5210c759eb20d99057882d1f Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 9 Feb 2024 11:49:41 -0500 Subject: [PATCH 049/141] mds folder changed to use PCU2 --- mds/apfMDS.cc | 6 +- mds/apfMDS.h | 7 +- mds/apfPM.cc | 2 +- mds/mdsCGNS.cc | 165 ++++++++++++++++++++++++++---------------------- mds/mds_apf.c | 1 - mds/mds_apf.h | 2 + mds/mds_net.c | 126 ++++++++++++++++++++++-------------- mds/mds_net.h | 11 ++++ mds/mds_order.c | 36 +++++++---- 9 files changed, 214 insertions(+), 142 deletions(-) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 23724e57e..a705b1f40 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -8,7 +8,7 @@ *******************************************************************************/ -#include +#include #include #include "apfMDS.h" #include "mds_apf.h" @@ -1028,13 +1028,13 @@ bool alignMdsMatches(Mesh2* in) if (!in->hasMatching()) return false; MeshMDS* m = static_cast(in); - return mds_align_matches(m->mesh); + return mds_align_matches2(in->getPCU()->GetCHandle(), m->mesh); } bool alignMdsRemotes(Mesh2* in) { MeshMDS* m = static_cast(in); - return mds_align_remotes(m->mesh); + return mds_align_remotes2(in->getPCU()->GetCHandle(), m->mesh); } void deriveMdsModel(Mesh2* in) diff --git a/mds/apfMDS.h b/mds/apfMDS.h index afb130fa2..3d0ac3410 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -35,7 +35,7 @@ // AJP: including apf.h for single define: CGNSBCMap // AJP: alternative is to allow common cgns base header // but trying to avoid that since it's not core functionality -#include +#include // struct gmi_model; @@ -50,6 +50,8 @@ class Field; /** \brief a map from global ids to vertex objects */ typedef std::map GlobalToVert; +typedef struct PCUHandle PCUHandle; + /** \brief create an empty MDS part \param model the geometric model interface @@ -199,9 +201,12 @@ int getMdsIndex(Mesh2* in, MeshEntity* e); MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index); Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); +Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); // names of mesh data to read from file: (VERTEX, VelocityX; CellCentre, Pressure) Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); +Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); + int gmshMajorVersion(const char* filename); diff --git a/mds/apfPM.cc b/mds/apfPM.cc index 201bb1e55..90039b153 100644 --- a/mds/apfPM.cc +++ b/mds/apfPM.cc @@ -8,7 +8,7 @@ *******************************************************************************/ -#include + #include "apfPM.h" #include #include diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index acdef4aae..fed33ff03 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -10,7 +10,8 @@ #include "apfShape.h" #include "gmi.h" /* this is for gmi_getline... */ #include -#include +//#include +#include #include #include #include "apfFieldData.h" @@ -200,14 +201,14 @@ struct MeshDataGroup }; template -void DebugParallelPrinter(std::ostream &out, Arg &&arg, Args &&... args) +void DebugParallelPrinter(PCUHandle h, std::ostream &out, Arg &&arg, Args &&... args) { // if constexpr (debugOutput) // probably will not get away with c++17 if (debugOutput) { - for (int i = 0; i < PCU_Comm_Peers(); i++) + for (int i = 0; i < PCU_Comm_Peers2(h); i++) { - if (i == PCU_Comm_Self()) + if (i == PCU_Comm_Self2(h)) { out << "Rank [" << i << "] " << std::forward(arg); //((out << ", " << std::forward(args)), ...); // probably will not get away with c++17 @@ -223,15 +224,15 @@ void DebugParallelPrinter(std::ostream &out, Arg &&arg, Args &&... args) } template -void Kill(const int fid, Args &&... args) +void Kill(PCUHandle h, const int fid, Args &&... args) { - DebugParallelPrinter(std::cout, "***** CGNS ERROR", args...); + DebugParallelPrinter(h, std::cout, "***** CGNS ERROR", args...); - if (PCU_Comm_Initialized()) + if (PCU_Comm_Initialized2(h)) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free(); + PCU_Comm_Free2(h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -244,15 +245,15 @@ void Kill(const int fid, Args &&... args) } } -void Kill(const int fid) +void Kill(PCUHandle h, const int fid) { - DebugParallelPrinter(std::cout, "***** CGNS ERROR"); + DebugParallelPrinter(h, std::cout, "***** CGNS ERROR"); - if (PCU_Comm_Initialized()) + if (PCU_Comm_Initialized2(h)) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free(); + PCU_Comm_Free2(h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -265,13 +266,13 @@ void Kill(const int fid) } } -auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const std::vector &, const apf::GlobalToVert &globalToVert) +auto ReadCGNSCoords(PCUHandle h, int cgid, int base, int zone, int ncoords, int nverts, const std::vector &, const apf::GlobalToVert &globalToVert) { // Read min required as defined by consecutive range // make one based as ReadElements makes zero based const cgsize_t lowest = globalToVert.begin()->first + 1; const cgsize_t highest = globalToVert.rbegin()->first + 1; - DebugParallelPrinter(std::cout, "From globalToVert ", lowest, highest, nverts); + DebugParallelPrinter(h, std::cout, "From globalToVert ", lowest, highest, nverts); cgsize_t range_min[3]; range_min[0] = lowest; @@ -301,7 +302,7 @@ auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const if (cg_coord_info(cgid, base, zone, d + 1, &datatype, &coord_names[d][0])) { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } const auto coord_name = std::string(coord_names[d].c_str()); //boost::algorithm::trim(coord_name); // can't be bothered including boost @@ -311,7 +312,7 @@ auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const ordinate.data())) { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } } // to be clear, indices passed back are global, zero based @@ -332,14 +333,14 @@ auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const return points; } -void SimpleElementPartition(std::vector &numberToReadPerProc, std::vector &startingIndex, int el_start /* one based */, int el_end, int numElements) +void SimpleElementPartition(PCUHandle h, std::vector &numberToReadPerProc, std::vector &startingIndex, int el_start /* one based */, int el_end, int numElements) { - numberToReadPerProc.resize(PCU_Comm_Peers(), 0); + numberToReadPerProc.resize(PCU_Comm_Peers2(h), 0); const cgsize_t blockSize = static_cast( std::floor(static_cast(numElements) / - static_cast(PCU_Comm_Peers()))); + static_cast(PCU_Comm_Peers2(h)))); - DebugParallelPrinter(std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers()); + DebugParallelPrinter(h, std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers2(h)); cgsize_t counter = 0; if (blockSize == 0 && numElements > 0) @@ -349,7 +350,7 @@ void SimpleElementPartition(std::vector &numberToReadPerProc, std::vec bool keepGoing = true; while (keepGoing) { - for (int p = 0; p < PCU_Comm_Peers() && keepGoing; p++) + for (int p = 0; p < PCU_Comm_Peers2(h) && keepGoing; p++) { if (counter + blockSize <= numElements) { @@ -366,11 +367,11 @@ void SimpleElementPartition(std::vector &numberToReadPerProc, std::vec } } } - DebugParallelPrinter(std::cout, "Sanity check: Counter", counter, "numElements", numElements); + DebugParallelPrinter(h, std::cout, "Sanity check: Counter", counter, "numElements", numElements); - DebugParallelPrinter(std::cout, "numberToReadPerProc for rank", PCU_Comm_Self(), "is:", numberToReadPerProc[PCU_Comm_Self()]); + DebugParallelPrinter(h, std::cout, "numberToReadPerProc for rank", PCU_Comm_Self2(h), "is:", numberToReadPerProc[PCU_Comm_Self2(h)]); - startingIndex.resize(PCU_Comm_Peers(), 0); + startingIndex.resize(PCU_Comm_Peers2(h), 0); startingIndex[0] = el_start; for (std::size_t i = 1; i < numberToReadPerProc.size(); i++) { @@ -378,36 +379,36 @@ void SimpleElementPartition(std::vector &numberToReadPerProc, std::vec startingIndex[i] = startingIndex[i - 1] + numberToReadPerProc[i - 1]; } - DebugParallelPrinter(std::cout, "Element start, end, numElements ", el_start, + DebugParallelPrinter(h, std::cout, "Element start, end, numElements ", el_start, el_end, numElements); - DebugParallelPrinter(std::cout, "startingIndex for rank", PCU_Comm_Self(), "is:", startingIndex[PCU_Comm_Self()]); + DebugParallelPrinter(h, std::cout, "startingIndex for rank", PCU_Comm_Self2(h), "is:", startingIndex[PCU_Comm_Self2(h)]); - DebugParallelPrinter(std::cout, "Returning from SimpleElementPartition \n"); + DebugParallelPrinter(h, std::cout, "Returning from SimpleElementPartition \n"); } using Pair = std::pair; using LocalElementRanges = std::vector; // one based -auto ReadElements(int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges) +auto ReadElements(PCUHandle h, int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges) { std::vector numberToReadPerProc; std::vector startingIndex; - SimpleElementPartition(numberToReadPerProc, startingIndex, el_start, el_end, numElements); + SimpleElementPartition(h, numberToReadPerProc, startingIndex, el_start, el_end, numElements); std::vector vertexIDs; - if (numberToReadPerProc[PCU_Comm_Self()] > 0) - vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self()] * verticesPerElement, + if (numberToReadPerProc[PCU_Comm_Self2(h)] > 0) + vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self2(h)] * verticesPerElement, -1234567); - const auto start = startingIndex[PCU_Comm_Self()]; - const auto end = startingIndex[PCU_Comm_Self()] + numberToReadPerProc[PCU_Comm_Self()] - 1; - DebugParallelPrinter(std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self()]); + const auto start = startingIndex[PCU_Comm_Self2(h)]; + const auto end = startingIndex[PCU_Comm_Self2h(h)] + numberToReadPerProc[PCU_Comm_Self2(h)] - 1; + DebugParallelPrinter(h, std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self2(h)]); // cgp_elements_read_data(cgid, base, zone, section, start, end, vertexIDs.data()); - if (numberToReadPerProc[PCU_Comm_Self()] > 0) + if (numberToReadPerProc[PCU_Comm_Self2(h)] > 0) { localElementRanges.push_back(std::make_pair(start, end)); } @@ -419,7 +420,7 @@ auto ReadElements(int cgid, int base, int zone, int section, int el_start /* one i = i - 1; } - return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self()]); + return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self2(h)]); } struct CGNSBCMeta @@ -497,7 +498,7 @@ struct BCInfo } else { - Kill(cgid, "GlobalToVert lookup problem", v); + Kill(m->getPCU()->GetCHandle(), cgid, "GlobalToVert lookup problem", v); } } @@ -805,7 +806,7 @@ struct BCInfo Add("CellCenter", bcName, bcTag); } else - Kill(cgid, "Unknown BC Type", cgnsLocation); + Kill(m->getPCU()->GetCHandle(), cgid, "Unknown BC Type", cgnsLocation); } else if (m->getDimension() == 2) // working with a 2D mesh { @@ -856,7 +857,7 @@ struct BCInfo } }; // namespace -void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector &bcInfos, const apf::GlobalToVert &globalToVert) +void ReadBCInfo(PCUHandle h, const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector &bcInfos, const apf::GlobalToVert &globalToVert) { // Read the BCS. std::vector bcMetas(nBocos); @@ -874,7 +875,7 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos if (cg_boco_info(cgid, base, zone, boco, &bcMeta.bocoName[0], &bcMeta.bocoType, &bcMeta.ptsetType, &bcMeta.npnts, bcMeta.normalindices.data(), &bcMeta.normalListSize, &bcMeta.normalDataType, &bcMeta.ndataset)) - Kill(cgid, "Failed cg_boco_info"); + Kill(h, cgid, "Failed cg_boco_info"); if (bcMeta.ptsetType == CGNS_ENUMV(PointList) || (bcMeta.ptsetType == CGNS_ENUMV(PointRange))) { @@ -882,7 +883,7 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos //boost::algorithm::trim(bcMeta.bocoName); // can't be bothered including boost if (cg_boco_gridlocation_read(cgid, base, zone, boco, &bcMeta.location)) - Kill(cgid, "Failed cg_boco_gridlocation_read"); + Kill(h, cgid, "Failed cg_boco_gridlocation_read"); bcMeta.locationName = cg_GridLocationName(bcMeta.location); @@ -891,7 +892,7 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos } else { - Kill(cgid, + Kill(h, cgid, "TODO: Can only work with " "PointList and PointRange BC " "types at the moment"); @@ -907,25 +908,25 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos if (bcMeta.locationName == "Vertex") // && cellDim == 1) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else if (bcMeta.locationName == "EdgeCenter") // && cellDim == 2) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else if (bcMeta.locationName == "FaceCenter") // && cellDim == 3) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else if (bcMeta.locationName == "CellCenter") // && cellDim == 3) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else - Kill(cgid, "Failed Location test for BC Type", bcMeta.locationName, + Kill(h, cgid, "Failed Location test for BC Type", bcMeta.locationName, cellDim); if (pointRange) @@ -1049,12 +1050,12 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos } } -apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &readMeshData) +apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &readMeshData) { static_assert(std::is_same::value, "cgsize_t not compiled as int"); int cgid = -1; - auto comm = PCU_Get_Comm(); + auto comm = PCU_Get_Comm2(h); cgp_mpi_comm(comm); cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); cgp_open(fname.c_str(), CGNS_ENUMV(CG_MODE_READ), &cgid); @@ -1111,7 +1112,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM if (ngrids > 1) { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } int ncoords = -1; cg_ncoords(cgid, base, zone, &ncoords); @@ -1128,7 +1129,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } int nBocos = -1; @@ -1157,14 +1158,14 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM cg_npe(elementType, &verticesPerElement); const auto readElementsAndVerts = [&](const apf::Mesh2::Type &type) { - const auto &ret = ReadElements(cgid, base, zone, section, el_start, el_end, numElements, verticesPerElement, localElementRanges); + const auto &ret = ReadElements(h, cgid, base, zone, section, el_start, el_end, numElements, verticesPerElement, localElementRanges); if (std::get<1>(ret) > 0) { const std::vector vertexIDs = std::get<0>(ret); std::vector vertexIDs_l(vertexIDs.begin(), vertexIDs.end()); localElements.emplace_back(apf::assemble(mesh, vertexIDs_l.data(), std::get<1>(ret), type, globalToVert)); // corresponding finalize below const auto nverts = sizes[0]; - const auto ordinates = ReadCGNSCoords(cgid, base, zone, ncoords, nverts, vertexIDs, globalToVert); + const auto ordinates = ReadCGNSCoords(h, cgid, base, zone, ncoords, nverts, vertexIDs, globalToVert); for (const auto &p : globalToVert) { @@ -1220,7 +1221,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM { std::cout << __LINE__ << " CGNS is dead " << " " << SupportedCGNSElementTypeToString(elementType) << std::endl; - Kill(cgid); + Kill(h, cgid); } } @@ -1229,13 +1230,13 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM std::cout << std::endl; std::cout << "Attempting to read BCS info " << " " << nBocos << std::endl; - ReadBCInfo(cgid, base, zone, nBocos, physDim, cellDim, nsections, bcInfos, globalToVert); + ReadBCInfo(h, cgid, base, zone, nBocos, physDim, cellDim, nsections, bcInfos, globalToVert); std::cout << std::endl; } int nsols = -1; if (cg_nsols(cgid, base, zone, &nsols)) - Kill(cgid, "1, ", nsols); + Kill(h, cgid, "1, ", nsols); std::vector meshData; if (nsols > 0) @@ -1247,11 +1248,11 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM char sname[33]; if (cg_sol_info(cgid, base, zone, ns, sname, &location)) - Kill(cgid, "2"); + Kill(h, cgid, "2"); int nflds = -1; if (cg_nfields(cgid, base, zone, ns, &nflds)) - Kill(cgid, "3"); + Kill(h, cgid, "3"); if (nflds > 0) { @@ -1262,7 +1263,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM char name[33]; if (cg_field_info(cgid, base, zone, ns, f, &datatype, name)) - Kill(cgid, "4"); + Kill(h, cgid, "4"); //std::cout << sname << " " << name << " " << f << " " << ns << " " << cg_DataTypeName(datatype) << " " << cg_GridLocationName(location) << std::endl; meshData.push_back(MeshData(ns, f, datatype, location, name)); @@ -1318,7 +1319,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM if (otherIndex != -1) group.insert(otherIndex, other); else - Kill(cgid, "Bad fail"); + Kill(h, cgid, "Bad fail"); group.insert(i, md); } } @@ -1439,7 +1440,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) field = apf::createFieldOn(mesh, md.name().c_str(), apf::MATRIX); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); double scalar(-123456.0); apf::Vector3 vector3(-123456.0, -123456.0, -123456.0); @@ -1456,7 +1457,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) apf::setMatrix(field, elem, 0, matrix3x3); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } mesh->end(it); @@ -1466,7 +1467,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM { if (cgp_field_read_data(cgid, base, zone, md.sIndex(i), md.fIndex(i), &range_min[0], &range_max[0], meshVals.data())) - Kill(cgid, "Failed cgp_field_read_data"); + Kill(h, cgid, "Failed cgp_field_read_data"); cgsize_t counter = lowest; for (cgsize_t it = 0; it < numToRead; it++) @@ -1498,7 +1499,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM apf::setMatrix(field, elem, 0, matrix3x3); } else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } counter++; } @@ -1509,7 +1510,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM } else { - Kill(cgid, "Don't know how to process this at the moment"); + Kill(h, cgid, "Don't know how to process this at the moment"); } } else if (md.location() == CGNS_ENUMV(CellCenter)) @@ -1535,7 +1536,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) field = apf::createField(mesh, md.name().c_str(), apf::MATRIX, apf::getConstant(dim)); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); double scalar(-123456.0); apf::Vector3 vector3(-123456.0, -123456.0, -123456.0); @@ -1552,7 +1553,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) apf::setMatrix(field, elem, 0, matrix3x3); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } mesh->end(it); @@ -1580,7 +1581,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM { if (cgp_field_read_data(cgid, base, zone, md.sIndex(i), md.fIndex(i), &range_min[0], &range_max[0], meshVals.data())) - Kill(cgid, "Failed cgp_field_read_data"); + Kill(h, cgid, "Failed cgp_field_read_data"); for (std::size_t it = 0; it < numToRead; it++) { @@ -1609,7 +1610,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM } else { - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } } } @@ -1620,19 +1621,19 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM } else { - Kill(cgid, "Don't know how to process this at the moment"); + Kill(h, cgid, "Don't know how to process this at the moment"); } } else { - Kill(cgid, "Don't know how to process this at the moment"); + Kill(h, cgid, "Don't know how to process this at the moment"); } } } } // free up memory - if (PCU_Comm_Initialized()) + if (PCU_Comm_Initialized2(h)) cgp_close(cgid); else cg_close(cgid); @@ -1711,10 +1712,10 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM return mesh; } // namespace -apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap) +apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap) { std::vector> meshData; - return DoIt(g, fname, cgnsBCMap, meshData); + return DoIt(h, g, fname, cgnsBCMap, meshData); } } // namespace @@ -1724,9 +1725,15 @@ namespace apf // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return loadMdsFromCGNS2(h, g, fname, cgnsBCMap, meshData); +} + +Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) { #ifdef HAVE_CGNS - Mesh2 *m = DoIt(g, fname, cgnsBCMap, meshData); + Mesh2 *m = DoIt(h, g, fname, cgnsBCMap, meshData); return m; #else Mesh2 *m = nullptr; @@ -1739,9 +1746,15 @@ Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMa // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return loadMdsFromCGNS2(h, g, fname, cgnsBCMap); +} + +Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) { #ifdef HAVE_CGNS - Mesh2 *m = DoIt(g, fname, cgnsBCMap); + Mesh2 *m = DoIt(h, g, fname, cgnsBCMap); return m; #else Mesh2 *m = nullptr; diff --git a/mds/mds_apf.c b/mds/mds_apf.c index d303dd47b..915f25d05 100644 --- a/mds/mds_apf.c +++ b/mds/mds_apf.c @@ -11,7 +11,6 @@ #include "mds_apf.h" #include #include -//#include #include struct mds_apf* mds_apf_create(struct gmi_model* model, int d, diff --git a/mds/mds_apf.h b/mds/mds_apf.h index 692024fdf..a453b9a0c 100644 --- a/mds/mds_apf.h +++ b/mds/mds_apf.h @@ -59,6 +59,8 @@ void mds_set_part(struct mds_apf* m, mds_id e, void* p); struct mds_tag* mds_number_verts_bfs(struct mds_apf* m); struct mds_apf* mds_reorder(struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers); +struct mds_apf* mds_reorder2(PCUHandle h, struct mds_apf* m, int ignore_peers, + struct mds_tag* vert_numbers); struct gmi_ent* mds_find_model(struct mds_apf* m, int dim, int id); int mds_model_dim(struct mds_apf* m, struct gmi_ent* model); diff --git a/mds/mds_net.c b/mds/mds_net.c index 2782ab368..8aa693d93 100644 --- a/mds/mds_net.c +++ b/mds/mds_net.c @@ -9,7 +9,7 @@ *******************************************************************************/ #include "mds_net.h" -#include +#include #include #include #include @@ -159,8 +159,8 @@ static void note_peer(struct mds_links* ln, int p) ++(ln->n[i]); } -static void for_type_net(struct mds_net* net, struct mds* m, - int t, void (*f)(mds_id i, struct mds_copy c, void* u), void* u) +static void for_type_net(PCUHandle h, struct mds_net* net, struct mds* m, + int t, void (*f)(PCUHandle h, mds_id i, struct mds_copy c, void* u), void* u) { mds_id i; int j; @@ -170,14 +170,14 @@ static void for_type_net(struct mds_net* net, struct mds* m, if (!cs) continue; for (j = 0; j < cs->n; ++j) - f(i, cs->c[j], u); + f(h, i, cs->c[j], u); } } -static void note_remote_link(mds_id i, struct mds_copy c, void* u) +static void note_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) { (void)i; - if (c.p != PCU_Comm_Self()) + if (c.p != PCU_Comm_Self2(h)) note_peer(u, c.p); } @@ -189,11 +189,11 @@ static void alloc_links(struct mds_links* ln) ln->l[i] = malloc(ln->n[i] * sizeof(unsigned)); } -static void take_remote_link(mds_id i, struct mds_copy c, void* u) +static void take_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; int pi; - if (PCU_Comm_Self() < c.p) { + if (PCU_Comm_Self2(h) < c.p) { pi = find_peer(ln, c.p); ln->l[pi][ln->n[pi]] = i; /* use n as position keeper */ @@ -201,68 +201,82 @@ static void take_remote_link(mds_id i, struct mds_copy c, void* u) } } -static void send_remote_link(mds_id i, struct mds_copy c, void* u) +static void send_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) { (void)i; (void)u; - if (PCU_Comm_Self() < c.p) - PCU_COMM_PACK(c.p, c.e); + if (PCU_Comm_Self2(h) < c.p) + PCU_COMM_PACK2(h, c.p, c.e); } -static void recv_links(struct mds_links* ln) +static void recv_links(PCUHandle h, struct mds_links* ln) { int from; mds_id* tmp; int pi; unsigned i; - from = PCU_Comm_Sender(); + from = PCU_Comm_Sender2(h); pi = find_peer(ln, from); - tmp = PCU_Comm_Extract(ln->n[pi] * sizeof(mds_id)); + tmp = PCU_Comm_Extract2(h, ln->n[pi] * sizeof(mds_id)); for (i = 0; i < ln->n[pi]; ++i) ln->l[pi][i] = mds_index(tmp[i]); } void mds_get_type_links(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_get_type_links2(h, net, m, t, ln); +} + +void mds_get_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln) { unsigned i; /* count remote links in np, p and n */ - for_type_net(net, m, t, note_remote_link, ln); + for_type_net(h, net, m, t, note_remote_link, ln); alloc_links(ln); for (i = 0; i < ln->np; ++i) - if (((unsigned)PCU_Comm_Self()) < ln->p[i]) + if (((unsigned)PCU_Comm_Self2(h)) < ln->p[i]) /* zero n's for behavior in take_copy */ ln->n[i] = 0; /* record indices in local order for owned boundaries */ - for_type_net(net, m, t, take_remote_link, ln); - PCU_Comm_Begin(); + for_type_net(h, net, m, t, take_remote_link, ln); + PCU_Comm_Begin2(h); /* send indices in local order for owned boundaries */ - for_type_net(net, m, t, send_remote_link, NULL); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + for_type_net(h, net, m, t, send_remote_link, NULL); + PCU_Comm_Send2(h); + while (PCU_Comm_Listen2(h)) /* recv indices in remote order for non-owned boundaries */ - recv_links(ln); + recv_links(h, ln); } void mds_set_type_links(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_set_type_links2(h, net, m, t, ln); +} + +void mds_set_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln) { unsigned i; unsigned j; unsigned* in; struct mds_copy c; - PCU_Comm_Begin(); + PCU_Comm_Begin2(h); for (i = 0; i < ln->np; ++i) { PCU_ALWAYS_ASSERT(ln->l); for (j = 0; j < ln->n[i]; ++j) - PCU_COMM_PACK(ln->p[i], ln->l[i][j]); + PCU_COMM_PACK2(h, ln->p[i], ln->l[i][j]); } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - c.p = PCU_Comm_Sender(); - PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self()); + PCU_Comm_Send2(h); + while (PCU_Comm_Listen2(h)) { + c.p = PCU_Comm_Sender2(h); + PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self2(h)); i = find_peer(ln, c.p); - in = PCU_Comm_Extract(ln->n[i] * sizeof(unsigned)); + in = PCU_Comm_Extract2(h, ln->n[i] * sizeof(unsigned)); for (j = 0; j < ln->n[i]; ++j) { c.e = mds_identify(t, in[j]); mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c); @@ -289,23 +303,23 @@ int mds_net_empty(struct mds_net* net) return 1; } -static void note_local_link(mds_id i, struct mds_copy c, void* u) +static void note_local_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) { - if (c.p == PCU_Comm_Self()) { + if (c.p == PCU_Comm_Self2(h)) { if (i < mds_index(c.e)) - note_peer(u, PCU_Comm_Self()); + note_peer(u, PCU_Comm_Self2(h)); else /* hack id to store self-receivers */ - note_peer(u, PCU_Comm_Peers()); + note_peer(u, PCU_Comm_Peers2(h)); } } -static void take_local_link(mds_id i, struct mds_copy c, void* u) +static void take_local_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; - int self = find_peer(ln, PCU_Comm_Self()); - int other = find_peer(ln, PCU_Comm_Peers()); + int self = find_peer(ln, PCU_Comm_Self2(h)); + int other = find_peer(ln, PCU_Comm_Peers2(h)); mds_id j = mds_index(c.e); - if ((PCU_Comm_Self() == c.p) && (i < j)) { + if ((PCU_Comm_Self2(h) == c.p) && (i < j)) { ln->l[self][ln->n[self]] = i; ln->l[other][ln->n[other]] = j; /* use ns as (redundant) position keepers */ @@ -316,33 +330,47 @@ static void take_local_link(mds_id i, struct mds_copy c, void* u) void mds_get_local_matches(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_get_local_matches2(h, net, m, t, ln); +} + +void mds_get_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln) { int self, other; - for_type_net(net, m, t, note_local_link, ln); - self = find_peer(ln, PCU_Comm_Self()); + for_type_net(h, net, m, t, note_local_link, ln); + self = find_peer(ln, PCU_Comm_Self2(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers()); + other = find_peer(ln, PCU_Comm_Peers2(h)); PCU_ALWAYS_ASSERT(ln->n[self] == ln->n[other]); ln->l[self] = malloc(ln->n[self] * sizeof(unsigned)); ln->l[other] = malloc(ln->n[other] * sizeof(unsigned)); ln->n[self] = 0; ln->n[other] = 0; - for_type_net(net, m, t, take_local_link, ln); + for_type_net(h, net, m, t, take_local_link, ln); } void mds_set_local_matches(struct mds_net* net, struct mds* m, int t, struct mds_links* ln) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_set_local_matches2(h, net, m, t, ln); +} + +void mds_set_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln) { int self, other; unsigned i; mds_id a, b; struct mds_copy c; - c.p = PCU_Comm_Self(); - self = find_peer(ln, PCU_Comm_Self()); + c.p = PCU_Comm_Self2(h); + self = find_peer(ln, PCU_Comm_Self2(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers()); + other = find_peer(ln, PCU_Comm_Peers2(h)); PCU_ALWAYS_ASSERT(ln->n != 0); PCU_ALWAYS_ASSERT(ln->n[self] == ln->n[other]); for (i = 0; i < ln->n[self]; ++i) { @@ -357,12 +385,18 @@ void mds_set_local_matches(struct mds_net* net, struct mds* m, } void mds_free_local_links(struct mds_links* ln) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_free_local_links2(h, ln); +} + +void mds_free_local_links2(PCUHandle h, struct mds_links* ln) { int self, other; - self = find_peer(ln, PCU_Comm_Self()); + self = find_peer(ln, PCU_Comm_Self2(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers()); + other = find_peer(ln, PCU_Comm_Peers2(h)); PCU_ALWAYS_ASSERT(ln->n != 0); ln->n[self] = ln->n[other] = 0; PCU_ALWAYS_ASSERT(ln->l != 0); diff --git a/mds/mds_net.h b/mds/mds_net.h index 69dec52b9..5cde35428 100644 --- a/mds/mds_net.h +++ b/mds/mds_net.h @@ -13,6 +13,8 @@ #include "mds.h" +typedef struct PCUHandle PCUHandle; + struct mds_copy { mds_id e; int p; @@ -52,16 +54,25 @@ void mds_add_copy(struct mds_net* net, struct mds* m, mds_id e, void mds_get_type_links(struct mds_net* net, struct mds* m, int t, struct mds_links* ln); +void mds_get_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln); void mds_set_type_links(struct mds_net* net, struct mds* m, int t, struct mds_links* ln); +void mds_set_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln); void mds_free_links(struct mds_links* ln); int mds_net_empty(struct mds_net* net); void mds_get_local_matches(struct mds_net* net, struct mds* m, int t, struct mds_links* ln); +void mds_get_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln); void mds_set_local_matches(struct mds_net* net, struct mds* m, int t, struct mds_links* ln); +void mds_set_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, + int t, struct mds_links* ln); void mds_free_local_links(struct mds_links* ln); +void mds_free_local_links2(PCUHandle h, struct mds_links* ln); #endif diff --git a/mds/mds_order.c b/mds/mds_order.c index d8ac16921..5f5034550 100644 --- a/mds/mds_order.c +++ b/mds/mds_order.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include struct queue { mds_id* e; @@ -183,7 +183,7 @@ static mds_id lookup(struct mds_tag* tag, mds_id old) } /* see apf/apfConvert.cc apf::Converter::createRemotes */ -static void rebuild_net(struct mds_net* net, +static void rebuild_net(PCUHandle h, struct mds_net* net, struct mds* m, struct mds_net* net2, struct mds* m2, @@ -197,7 +197,7 @@ static void rebuild_net(struct mds_net* net, struct mds_copies* cs; struct mds_copy c; int i; - PCU_Comm_Begin(); + PCU_Comm_Begin2(h); for (d = 0; d <= m->d; ++d) for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) { cs = mds_get_copies(net, e); @@ -206,16 +206,16 @@ static void rebuild_net(struct mds_net* net, ne = lookup(new_of, e); for (i = 0; i < cs->n; ++i) { ce = cs->c[i].e; - PCU_COMM_PACK(cs->c[i].p, ce); - PCU_COMM_PACK(cs->c[i].p, ne); + PCU_COMM_PACK2(h, cs->c[i].p, ce); + PCU_COMM_PACK2(h, cs->c[i].p, ne); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - c.p = PCU_Comm_Sender(); - while (!PCU_Comm_Unpacked()) { - PCU_COMM_UNPACK(ce); - PCU_COMM_UNPACK(ne); + PCU_Comm_Send2(h); + while (PCU_Comm_Listen2(h)) { + c.p = PCU_Comm_Sender2(h); + while (!PCU_Comm_Unpacked2(h)) { + PCU_COMM_UNPACK2(h, ce); + PCU_COMM_UNPACK2(h, ne); c.e = ne; nce = lookup(new_of, ce); mds_add_copy(net2, m2, nce, c); @@ -380,6 +380,7 @@ static void rebuild_parts( } static struct mds_apf* rebuild( + PCUHandle h, struct mds_apf* m, struct mds_tag* new_of, int ignore_peers) @@ -394,10 +395,10 @@ static struct mds_apf* rebuild( rebuild_coords(m, m2, old_of); rebuild_parts(m, m2, old_of); if (!ignore_peers) { - rebuild_net(&m->remotes, &m->mds, + rebuild_net(h, &m->remotes, &m->mds, &m2->remotes, &m2->mds, new_of); - rebuild_net(&m->matches, &m->mds, + rebuild_net(h, &m->matches, &m->mds, &m2->matches, &m2->mds, new_of); } @@ -407,12 +408,19 @@ static struct mds_apf* rebuild( struct mds_apf* mds_reorder(struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers) +{ + PCUHandle h = PCU_Get_Global_Handle(); + return mds_reorder2(h, m, ignore_peers, vert_numbers); +} + +struct mds_apf* mds_reorder2(PCUHandle h, struct mds_apf* m, int ignore_peers, + struct mds_tag* vert_numbers) { struct mds_tag* tag; struct mds_apf* m2; tag = vert_numbers; number_other_ents(m, tag); - m2 = rebuild(m, tag, ignore_peers); + m2 = rebuild(h, m, tag, ignore_peers); mds_apf_destroy(m); return m2; } From 5206e77151fb9eafdebdcd6e5af341d04678762b Mon Sep 17 00:00:00 2001 From: flagdanger Date: Sun, 11 Feb 2024 23:38:25 -0500 Subject: [PATCH 050/141] Last mds fixes (apfMDS.cc still has some issues) --- mds/apfMDS.cc | 13 +++++++------ mds/mds_smb.c | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index a705b1f40..82d48015c 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -8,6 +8,7 @@ *******************************************************************************/ +#include #include #include #include "apfMDS.h" @@ -198,7 +199,7 @@ class MeshMDS : public Mesh2 MeshMDS(gmi_model* m, const char* pathname) { init(apf::getLagrange(1)); - mesh = mds_read_smb(m, pathname, 0, this); + mesh = mds_read_smb2(this->getPCU()->GetCHandle(), m, pathname, 0, this); isMatched = this->getPCU()->Or(!mds_net_empty(&mesh->matches)); ownsModel = true; } @@ -584,7 +585,7 @@ class MeshMDS : public Mesh2 void writeNative(const char* fileName) { double t0 = pcu::Time(); - mesh = mds_write_smb(mesh, fileName, 0, this); + mesh = mds_write_smb2(this->getPCU()->GetCHandle(), mesh, fileName, 0, this); double t1 = pcu::Time(); if (!this->getPCU()->Self()) lion_oprint(1,"mesh %s written in %f seconds\n", fileName, t1 - t0); @@ -964,8 +965,8 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) } else { vert_nums = mds_number_verts_bfs(m->mesh); } - m->mesh = mds_reorder(m->mesh, 0, vert_nums); - if (!m->getPCU()->Self()) + m->mesh = mds_reorder2(mesh->getPCU()->GetCHandle(), m->mesh, 0, vert_nums); + if (!mesh->getPCU()->Self()) lion_oprint(1,"mesh reordered in %f seconds\n", pcu::Time()-t0); } @@ -1304,7 +1305,7 @@ Mesh2* loadMdsPart(gmi_model* model, const char* meshfile) { MeshMDS* m = new MeshMDS(); m->init(apf::getLagrange(1)); - m->mesh = mds_read_smb(model, meshfile, 1, m); + m->mesh = mds_read_smb2(m->getPCU()->GetCHandle(), model, meshfile, 1, m); m->isMatched = false; m->ownsModel = true; initResidence(m, m->getDimension()); @@ -1314,7 +1315,7 @@ Mesh2* loadMdsPart(gmi_model* model, const char* meshfile) void writeMdsPart(Mesh2* in, const char* meshfile) { MeshMDS* m = static_cast(in); - m->mesh = mds_write_smb(m->mesh, meshfile, 1, m); + m->mesh = mds_write_smb2(m->getPCU()->GetCHandle(), m->mesh, meshfile, 1, m); } diff --git a/mds/mds_smb.c b/mds/mds_smb.c index d91ae751b..def9d9fd9 100644 --- a/mds/mds_smb.c +++ b/mds/mds_smb.c @@ -212,22 +212,22 @@ static void write_conn(struct pcu_file* f, struct mds_apf* m) } } -static void read_remotes(struct pcu_file* f, struct mds_apf* m, +static void read_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; read_links(f, &ln); if (!ignore_peers) - mds_set_type_links(&m->remotes, &m->mds, MDS_VERTEX, &ln); + mds_set_type_links2(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); mds_free_links(&ln); } -static void write_remotes(struct pcu_file* f, struct mds_apf* m, +static void write_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; if (!ignore_peers) - mds_get_type_links(&m->remotes, &m->mds, MDS_VERTEX, &ln); + mds_get_type_links2(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); write_links(f, &ln); mds_free_links(&ln); } @@ -510,53 +510,53 @@ static void write_tags(struct pcu_file* f, struct mds_apf* m) free(sizes); } -static void read_type_matches(struct pcu_file* f, struct mds_apf* m, int t, +static void read_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int t, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; read_links(f, &ln); if (!ignore_peers) - mds_set_local_matches(&m->matches, &m->mds, t, &ln); - mds_free_local_links(&ln); + mds_set_local_matches2(h, &m->matches, &m->mds, t, &ln); + mds_free_local_links2(h, &ln); if (!ignore_peers) - mds_set_type_links(&m->matches, &m->mds, t, &ln); + mds_set_type_links2(h, &m->matches, &m->mds, t, &ln); mds_free_links(&ln); } -static void write_type_matches(struct pcu_file* f, struct mds_apf* m, int t, +static void write_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int t, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; if (!ignore_peers) { - mds_get_type_links(&m->matches, &m->mds, t, &ln); - mds_get_local_matches(&m->matches, &m->mds, t, &ln); + mds_get_type_links2(h, &m->matches, &m->mds, t, &ln); + mds_get_local_matches2(h, &m->matches, &m->mds, t, &ln); } write_links(f, &ln); mds_free_links(&ln); } -static void read_matches_old(struct pcu_file* f, struct mds_apf* m, +static void read_matches_old(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; for (t = 0; t < MDS_HEXAHEDRON; ++t) - read_type_matches(f, m, t, ignore_peers); + read_type_matches(h, f, m, t, ignore_peers); } -static void read_matches_new(struct pcu_file* f, struct mds_apf* m, +static void read_matches_new(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; for (t = 0; t < SMB_TYPES; ++t) - read_type_matches(f, m, smb2mds(t), ignore_peers); + read_type_matches(h, f, m, smb2mds(t), ignore_peers); } -static void write_matches(struct pcu_file* f, struct mds_apf* m, +static void write_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; for (t = 0; t < SMB_TYPES; ++t) - write_type_matches(f, m, smb2mds(t), ignore_peers); + write_type_matches(h, f, m, smb2mds(t), ignore_peers); } static struct mds_apf* read_smb(PCUHandle h, struct gmi_model* model, const char* filename, @@ -592,13 +592,13 @@ static struct mds_apf* read_smb(PCUHandle h, struct gmi_model* model, const char for (pj = 0; pj < 2; ++pj) m->param[pi][pj] = 0.0; } } - read_remotes(f, m, ignore_peers); + read_remotes(h, f, m, ignore_peers); read_class(f, m); read_tags(f, m); if (version >= 4) - read_matches_new(f, m, ignore_peers); + read_matches_new(h, f, m, ignore_peers); else if (version >= 3) - read_matches_old(f, m, ignore_peers); + read_matches_old(h, f, m, ignore_peers); if (version >= 5) mds_read_smb_meta(f, m, apf_mesh); pcu_fclose(f); @@ -628,10 +628,10 @@ static void write_smb(PCUHandle h, struct mds_apf* m, const char* filename, pcu_write_unsigneds(f, n, SMB_TYPES); write_conn(f, m); write_coords(f, m); - write_remotes(f, m, ignore_peers); + write_remotes(h, f, m, ignore_peers); write_class(f, m); write_tags(f, m); - write_matches(f, m, ignore_peers); + write_matches(h, f, m, ignore_peers); mds_write_smb_meta(f, apf_mesh); pcu_fclose(f); } @@ -774,11 +774,11 @@ struct mds_apf* mds_write_smb2(PCUHandle h, struct mds_apf* m, const char* pathn int zip; if (ignore_peers && (!is_compact(m))) { if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); - m = mds_reorder(m, 1, mds_number_verts_bfs(m)); + m = mds_reorder2(h, m, 1, mds_number_verts_bfs(m)); } if ((!ignore_peers) && PCU_Or2(h, !is_compact(m))) { if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); - m = mds_reorder(m, 0, mds_number_verts_bfs(m)); + m = mds_reorder2(h, m, 0, mds_number_verts_bfs(m)); } filename = handle_path(h, pathname, 1, &zip, ignore_peers); write_smb(h, m, filename, zip, ignore_peers, apf_mesh); From 8e2cdb055e69f59fc5c39c73ae8b653e008a6425 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Sun, 11 Feb 2024 23:44:21 -0500 Subject: [PATCH 051/141] Getting rid of include PCU.h in spots I missed --- mds/mds.c | 1 - mds/mdsCGNS.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/mds/mds.c b/mds/mds.c index ed2e0cc51..a896da704 100644 --- a/mds/mds.c +++ b/mds/mds.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index fed33ff03..59e7df0d2 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -10,7 +10,6 @@ #include "apfShape.h" #include "gmi.h" /* this is for gmi_getline... */ #include -//#include #include #include #include From 221a982c4b25c7027c3d233a9c27dc5a6f061826 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 00:13:36 -0500 Subject: [PATCH 052/141] All parma changes I can do (lots of files need some workaround) --- parma/diffMC/parma_centroidDiffuser.cc | 1 - parma/diffMC/parma_centroidSelector.cc | 1 - parma/diffMC/parma_centroids.cc | 1 - parma/diffMC/parma_components.cc | 1 - parma/diffMC/parma_dcpart.cc | 1 - parma/diffMC/parma_dijkstra.cc | 1 - parma/diffMC/parma_edgeEqVtxSelector.cc | 1 - parma/diffMC/parma_elmBalancer.cc | 1 - parma/diffMC/parma_elmLtVtxEdgeBalancer.cc | 1 - parma/diffMC/parma_elmLtVtxEdgeSelector.cc | 1 - parma/diffMC/parma_elmLtVtxEdgeTargets.cc | 1 - parma/diffMC/parma_ghost.cc | 1 - parma/diffMC/parma_ghostElement.cc | 1 - parma/diffMC/parma_ghostMPAS.cc | 1 - parma/diffMC/parma_ghostOwner.cc | 1 - parma/diffMC/parma_ghostOwner.h | 1 - parma/diffMC/parma_graphDist.cc | 1 - parma/diffMC/parma_ltSelector.cc | 1 - parma/diffMC/parma_preserveTargets.cc | 1 - parma/diffMC/parma_shapeSelector.cc | 1 - parma/diffMC/parma_step.cc | 1 - parma/diffMC/parma_vtxBalancer.cc | 1 - parma/diffMC/parma_vtxEdgeElmBalancer.cc | 1 - parma/diffMC/parma_vtxEdgeTargets.cc | 1 - parma/diffMC/parma_vtxElmBalancer.cc | 1 - parma/diffMC/parma_vtxSelector.cc | 1 - parma/diffMC/parma_weightSideTargets.cc | 1 - parma/rib/parma_mesh_rib.cc | 5 ++--- 28 files changed, 2 insertions(+), 30 deletions(-) diff --git a/parma/diffMC/parma_centroidDiffuser.cc b/parma/diffMC/parma_centroidDiffuser.cc index cdfa82c6a..a15a294d9 100644 --- a/parma/diffMC/parma_centroidDiffuser.cc +++ b/parma/diffMC/parma_centroidDiffuser.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "parma_step.h" diff --git a/parma/diffMC/parma_centroidSelector.cc b/parma/diffMC/parma_centroidSelector.cc index d3ffd5711..63e37f07d 100644 --- a/parma/diffMC/parma_centroidSelector.cc +++ b/parma/diffMC/parma_centroidSelector.cc @@ -1,4 +1,3 @@ -#include "PCU.h" #include #include "parma_selector.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_centroids.cc b/parma/diffMC/parma_centroids.cc index 482e31a5d..aee591337 100644 --- a/parma/diffMC/parma_centroids.cc +++ b/parma/diffMC/parma_centroids.cc @@ -1,5 +1,4 @@ #include -#include #include "parma_centroids.h" #include "parma_sides.h" diff --git a/parma/diffMC/parma_components.cc b/parma/diffMC/parma_components.cc index 4f093587c..37497a537 100644 --- a/parma/diffMC/parma_components.cc +++ b/parma/diffMC/parma_components.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include "parma_components.h" #include "parma_meshaux.h" #include "parma_convert.h" diff --git a/parma/diffMC/parma_dcpart.cc b/parma/diffMC/parma_dcpart.cc index 00d837c4a..789102fad 100644 --- a/parma/diffMC/parma_dcpart.cc +++ b/parma/diffMC/parma_dcpart.cc @@ -1,4 +1,3 @@ -#include "PCU.h" #include "parma_dcpart.h" #include "parma_commons.h" #include "parma_meshaux.h" diff --git a/parma/diffMC/parma_dijkstra.cc b/parma/diffMC/parma_dijkstra.cc index 22e27c118..30d2b2966 100644 --- a/parma/diffMC/parma_dijkstra.cc +++ b/parma/diffMC/parma_dijkstra.cc @@ -1,6 +1,5 @@ #include #include -#include #include #include "parma_dijkstra.h" #include "parma_meshaux.h" diff --git a/parma/diffMC/parma_edgeEqVtxSelector.cc b/parma/diffMC/parma_edgeEqVtxSelector.cc index 710014376..1d517774b 100644 --- a/parma/diffMC/parma_edgeEqVtxSelector.cc +++ b/parma/diffMC/parma_edgeEqVtxSelector.cc @@ -1,6 +1,5 @@ #include #include -#include #include "parma_vtxSelector.h" #include "parma_targets.h" #include "parma_weights.h" diff --git a/parma/diffMC/parma_elmBalancer.cc b/parma/diffMC/parma_elmBalancer.cc index 8c2a32efb..ae477984d 100644 --- a/parma/diffMC/parma_elmBalancer.cc +++ b/parma/diffMC/parma_elmBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "parma_step.h" diff --git a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc index c23c9bfe1..05d15c656 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_balancer.h" diff --git a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc index e3b0f6918..681d43880 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc @@ -1,6 +1,5 @@ #include #include -#include #include "parma_vtxSelector.h" #include "parma_targets.h" #include "parma_weights.h" diff --git a/parma/diffMC/parma_elmLtVtxEdgeTargets.cc b/parma/diffMC/parma_elmLtVtxEdgeTargets.cc index 64ca0a2b6..067d9a81b 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeTargets.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_ghost.cc b/parma/diffMC/parma_ghost.cc index ee9f2b7f3..62f653453 100644 --- a/parma/diffMC/parma_ghost.cc +++ b/parma/diffMC/parma_ghost.cc @@ -1,5 +1,4 @@ #include -#include #include "parma.h" #include "parma_balancer.h" #include "parma_step.h" diff --git a/parma/diffMC/parma_ghostElement.cc b/parma/diffMC/parma_ghostElement.cc index dee8ef90d..f33dde4bd 100644 --- a/parma/diffMC/parma_ghostElement.cc +++ b/parma/diffMC/parma_ghostElement.cc @@ -1,5 +1,4 @@ #include -#include #include "parma.h" #include "parma_balancer.h" #include "parma_step.h" diff --git a/parma/diffMC/parma_ghostMPAS.cc b/parma/diffMC/parma_ghostMPAS.cc index d4840bdb5..2e06d9e65 100644 --- a/parma/diffMC/parma_ghostMPAS.cc +++ b/parma/diffMC/parma_ghostMPAS.cc @@ -1,5 +1,4 @@ #include -#include #include #include "parma.h" #include "parma_balancer.h" diff --git a/parma/diffMC/parma_ghostOwner.cc b/parma/diffMC/parma_ghostOwner.cc index 853d6b757..ee1bcb79d 100644 --- a/parma/diffMC/parma_ghostOwner.cc +++ b/parma/diffMC/parma_ghostOwner.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_ghostOwner.h" diff --git a/parma/diffMC/parma_ghostOwner.h b/parma/diffMC/parma_ghostOwner.h index b770acc10..767e711c7 100644 --- a/parma/diffMC/parma_ghostOwner.h +++ b/parma/diffMC/parma_ghostOwner.h @@ -1,7 +1,6 @@ #ifndef PARMA_GHOSTOWNER_H #define PARMA_GHOSTOWNER_H -#include #include #include diff --git a/parma/diffMC/parma_graphDist.cc b/parma/diffMC/parma_graphDist.cc index 804dfff59..983830c4c 100644 --- a/parma/diffMC/parma_graphDist.cc +++ b/parma/diffMC/parma_graphDist.cc @@ -1,5 +1,4 @@ #include -#include #include #include "parma_graphDist.h" #include "parma_dijkstra.h" diff --git a/parma/diffMC/parma_ltSelector.cc b/parma/diffMC/parma_ltSelector.cc index ebc3f91bd..5ab98a9e5 100644 --- a/parma/diffMC/parma_ltSelector.cc +++ b/parma/diffMC/parma_ltSelector.cc @@ -1,6 +1,5 @@ #include #include -#include #include #include "parma_vtxSelector.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_preserveTargets.cc b/parma/diffMC/parma_preserveTargets.cc index 387170c48..5214d480c 100644 --- a/parma/diffMC/parma_preserveTargets.cc +++ b/parma/diffMC/parma_preserveTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_shapeSelector.cc b/parma/diffMC/parma_shapeSelector.cc index e1d1af4e0..967a9f5a3 100644 --- a/parma/diffMC/parma_shapeSelector.cc +++ b/parma/diffMC/parma_shapeSelector.cc @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/parma/diffMC/parma_step.cc b/parma/diffMC/parma_step.cc index 846a6c8ee..5064189d6 100644 --- a/parma/diffMC/parma_step.cc +++ b/parma/diffMC/parma_step.cc @@ -1,4 +1,3 @@ -#include #include #include "parma_step.h" #include "parma_sides.h" diff --git a/parma/diffMC/parma_vtxBalancer.cc b/parma/diffMC/parma_vtxBalancer.cc index 0f5ac0aa8..599b39e1b 100644 --- a/parma/diffMC/parma_vtxBalancer.cc +++ b/parma/diffMC/parma_vtxBalancer.cc @@ -1,4 +1,3 @@ -#include #include "parma.h" #include "parma_step.h" #include "parma_balancer.h" diff --git a/parma/diffMC/parma_vtxEdgeElmBalancer.cc b/parma/diffMC/parma_vtxEdgeElmBalancer.cc index 383340862..f72230362 100644 --- a/parma/diffMC/parma_vtxEdgeElmBalancer.cc +++ b/parma/diffMC/parma_vtxEdgeElmBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_balancer.h" diff --git a/parma/diffMC/parma_vtxEdgeTargets.cc b/parma/diffMC/parma_vtxEdgeTargets.cc index ebc51aded..177ddeba8 100644 --- a/parma/diffMC/parma_vtxEdgeTargets.cc +++ b/parma/diffMC/parma_vtxEdgeTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_vtxElmBalancer.cc b/parma/diffMC/parma_vtxElmBalancer.cc index 11185b5ce..ac0382474 100644 --- a/parma/diffMC/parma_vtxElmBalancer.cc +++ b/parma/diffMC/parma_vtxElmBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_balancer.h" diff --git a/parma/diffMC/parma_vtxSelector.cc b/parma/diffMC/parma_vtxSelector.cc index aed10ec4b..d28197e24 100644 --- a/parma/diffMC/parma_vtxSelector.cc +++ b/parma/diffMC/parma_vtxSelector.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "parma_vtxSelector.h" diff --git a/parma/diffMC/parma_weightSideTargets.cc b/parma/diffMC/parma_weightSideTargets.cc index f721cf116..aa8e9c938 100644 --- a/parma/diffMC/parma_weightSideTargets.cc +++ b/parma/diffMC/parma_weightSideTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/rib/parma_mesh_rib.cc b/parma/rib/parma_mesh_rib.cc index ed0dff41f..31915e86d 100644 --- a/parma/rib/parma_mesh_rib.cc +++ b/parma/rib/parma_mesh_rib.cc @@ -1,4 +1,3 @@ -#include #include "parma_rib.h" #include #include @@ -63,7 +62,7 @@ class RibSplitter : public apf::Splitter virtual apf::Migration* split(apf::MeshTag* weights, double, int multiple) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); int depth; for (depth = 0; (1 << depth) < multiple; ++depth); PCU_ALWAYS_ASSERT((1 << depth) == multiple); @@ -75,7 +74,7 @@ class RibSplitter : public apf::Splitter int p = plan->sending(e); plan->send(e, p + offset); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); if (!mesh->getPCU()->Self()) lion_oprint(1,"planned RIB factor %d in %f seconds\n", multiple, t1 - t0); From 344e3c49c63f098e3606e773ca8da12560d177c5 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 00:20:23 -0500 Subject: [PATCH 053/141] small pcu change --- pcu/pcu_io.c | 1 - 1 file changed, 1 deletion(-) diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index ebdc20974..b22ab51fc 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_io.h" -//#include "PCU.h" #include "PCU2.h" #include "noto_malloc.h" #include "pcu_buffer.h" From 5ee39e55812011b2b8d6aebf5dec075f559e95e1 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 11:34:19 -0500 Subject: [PATCH 054/141] pcu_getglobal changes --- apf/apfMesh.cc | 2 +- mds/apfMDS.cc | 1 + pcu/PCUObj.h | 5 +++++ pcu/pcu.cc | 7 +++++-- test/repartition.cc | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 3e140864b..069c62c29 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -168,7 +168,7 @@ void Mesh::init(FieldShape* s) baseP->init("coordinates",this,s,data); data->init(baseP); hasFrozenFields = false; - pcu_ = PCU_GetGlobal(); + pcu_ = pcu::PCU_GetGlobal(); } Mesh::~Mesh() diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 82d48015c..d07a7c8a2 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -973,6 +973,7 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) { double t0 = pcu::Time(); + //PCU_ALWAYS_ASSERT(PCU_GetGlobal() != m->getPCU()); int self = PCU_Comm_Self(); int outputPartCount = PCU_Comm_Peers(); apf::Expand expand(inputPartCount, outputPartCount); diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index 2f81dbd2c..fd3e4f84b 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -105,6 +105,8 @@ void Protect() noexcept; /*MPI_Wtime() equivalent*/ [[nodiscard]] double Time() noexcept; +PCU* PCU_GetGlobal(); + /* explicit instantiations of template functions */ #define PCU_EXPL_INST_DECL(T) \ extern template void PCU::Add(T * p, size_t n) noexcept; \ @@ -122,5 +124,8 @@ PCU_EXPL_INST_DECL(double) #undef PCU_EXPL_INST_DECL } // namespace pcu + #undef PCU_FORMAT_ATTRIBUTE #endif // SCOREC_PCU_PCUOBJ_H + + diff --git a/pcu/pcu.cc b/pcu/pcu.cc index 781d5219b..d50937364 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu.cc @@ -35,6 +35,10 @@ #include static pcu::PCU *global_pcu = nullptr; +namespace pcu { + pcu::PCU* PCU_GetGlobal() { return global_pcu; } +} + extern "C" { /** \brief Initializes the PCU library. \details This function must be called by all MPI processes before @@ -567,11 +571,10 @@ double PCU_GetMem(void) { return pcu::GetMem(); } /** \brief Return the global PCU */ -pcu::PCU* PCU_GetGlobal(void) { return global_pcu; } PCUHandle PCU_Get_Global_Handle(void) { PCUHandle h; - h.ptr = static_cast(PCU_GetGlobal()); + h.ptr = static_cast(pcu::PCU_GetGlobal()); return h; } diff --git a/test/repartition.cc b/test/repartition.cc index 4325116f0..e10e7d875 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -105,7 +105,7 @@ int main(int argc, char** argv) if (isOriginal) m = apf::loadMdsMesh(g, meshFile); switchToAll(); - m = apf::expandMdsMesh(m, g, inputPartCount); + m = apf::MdsMesh(m, g, inputPartCount); balance(m); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); From cb971df0fce233dc61ba8a4a561ff815675df665 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 16:57:48 -0500 Subject: [PATCH 055/141] apfCap fix --- apf_cap/apfCAP.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/apf_cap/apfCAP.cc b/apf_cap/apfCAP.cc index a3e72a6d5..2cc0b9cc0 100644 --- a/apf_cap/apfCAP.cc +++ b/apf_cap/apfCAP.cc @@ -1,5 +1,4 @@ #include "apfCAP.h" -#include #include #include #include From 2bb8171ba8779fc7e4a631336ac45bb48d1c8a23 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 16:58:10 -0500 Subject: [PATCH 056/141] crv fixes --- crv/crvAdapt.cc | 1 - crv/crvShape.cc | 1 - crv/crvShapeFixer.cc | 1 - crv/crvShapeFixer.h | 1 - crv/crvVtk.cc | 1 - 5 files changed, 5 deletions(-) diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc index a6bd5fa41..2b0121b47 100644 --- a/crv/crvAdapt.cc +++ b/crv/crvAdapt.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include namespace crv { diff --git a/crv/crvShape.cc b/crv/crvShape.cc index 4ac1a99f4..73f649e29 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include "crv.h" #include "crvAdapt.h" diff --git a/crv/crvShapeFixer.cc b/crv/crvShapeFixer.cc index b05fac91f..a063ca2ec 100644 --- a/crv/crvShapeFixer.cc +++ b/crv/crvShapeFixer.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "crvShapeFixer.h" #include "crvTables.h" #include "maTables.h" diff --git a/crv/crvShapeFixer.h b/crv/crvShapeFixer.h index 4757675a7..1233e4eae 100644 --- a/crv/crvShapeFixer.h +++ b/crv/crvShapeFixer.h @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "crvAdapt.h" #include "maOperator.h" #include "maEdgeSwap.h" diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc index 236126f7f..5258aca1b 100644 --- a/crv/crvVtk.cc +++ b/crv/crvVtk.cc @@ -6,7 +6,6 @@ */ #include "crv.h" -#include "PCU.h" #include "apfDynamicVector.h" #include "apfFieldData.h" #include "apfMDS.h" From 23bf88934b66b0e79c759513c368813bccf45b8b Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 16:58:29 -0500 Subject: [PATCH 057/141] dsp pcu fix --- dsp/dspGraphDistance.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dsp/dspGraphDistance.cc b/dsp/dspGraphDistance.cc index 853104f0a..03dd79e56 100644 --- a/dsp/dspGraphDistance.cc +++ b/dsp/dspGraphDistance.cc @@ -1,5 +1,4 @@ #include "dspGraphDistance.h" -#include #include namespace dsp { @@ -71,7 +70,7 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, } m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(sv); + m->getPCU()->Unpack(sv); if (!apf::isNumbered(dst, sv, 0, 0)) { apf::number(dst, sv, 0, 0, layer + 1); push(vs, sv); From 60e0b6fab1de59c4a4630c1e673bf3945f647b1b Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 16:58:56 -0500 Subject: [PATCH 058/141] gmi cap and sim pcu fixes --- gmi_cap/gmi_cap.cc | 1 - gmi_sim/gmi_sim.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/gmi_cap/gmi_cap.cc b/gmi_cap/gmi_cap.cc index 851c331ea..12c393cd3 100644 --- a/gmi_cap/gmi_cap.cc +++ b/gmi_cap/gmi_cap.cc @@ -7,7 +7,6 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include #include "gmi_cap.h" #include #include diff --git a/gmi_sim/gmi_sim.cc b/gmi_sim/gmi_sim.cc index b89375746..e79b25d6b 100644 --- a/gmi_sim/gmi_sim.cc +++ b/gmi_sim/gmi_sim.cc @@ -7,7 +7,6 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include #include "gmi_sim.h" #include #include From 63af137eaa10529c1662bde99469fdda33200f35 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 16:59:30 -0500 Subject: [PATCH 059/141] ma pcu fixes --- ma/ma.cc | 1 - ma/maAdapt.cc | 6 +++--- ma/maBalance.cc | 1 - ma/maCoarsen.cc | 1 - ma/maCrawler.cc | 5 ++--- ma/maDBG.cc | 1 - ma/maExtrude.cc | 1 - ma/maLayer.cc | 1 - ma/maLayerCoarsen.cc | 1 - ma/maLayerRefine.cc | 1 - ma/maLayerSnap.cc | 1 - ma/maMatchedCollapse.cc | 1 - ma/maMatchedSnapper.cc | 1 - ma/maMesh.cc | 1 - ma/maRefine.cc | 1 - ma/maShape.cc | 1 - ma/maSize.cc | 1 - ma/maSnap.cc | 1 - ma/maTetrahedronize.cc | 1 - 19 files changed, 5 insertions(+), 23 deletions(-) diff --git a/ma/ma.cc b/ma/ma.cc index ee2f6ac2e..23bb1ff41 100644 --- a/ma/ma.cc +++ b/ma/ma.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "ma.h" #include "maAdapt.h" #include "maCoarsen.h" diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index ec64bfcd3..6beb7bad8 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -246,9 +246,9 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) m->getPCU()->Send(); bool ok = true; while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); bool value; - PCU_COMM_UNPACK(value); + m->getPCU()->Unpack(value); if(value != getFlag(a,e,flag)) ok = false; } @@ -514,7 +514,7 @@ void syncFlag(Adapt* a, int dimension, int flag) m->end(it); m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); setFlag(a,e,flag); } delete sh; diff --git a/ma/maBalance.cc b/ma/maBalance.cc index 7fdefe990..f446e44a6 100644 --- a/ma/maBalance.cc +++ b/ma/maBalance.cc @@ -1,4 +1,3 @@ -#include #include "maBalance.h" #include "maAdapt.h" #include diff --git a/ma/maCoarsen.cc b/ma/maCoarsen.cc index 419393946..a8c2ece3e 100644 --- a/ma/maCoarsen.cc +++ b/ma/maCoarsen.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maCoarsen.h" #include "maAdapt.h" #include "maCollapse.h" diff --git a/ma/maCrawler.cc b/ma/maCrawler.cc index 79b66e8e6..8f97407d4 100644 --- a/ma/maCrawler.cc +++ b/ma/maCrawler.cc @@ -1,4 +1,3 @@ -#include #include "maCrawler.h" #include "maAdapt.h" #include "maLayer.h" @@ -26,7 +25,7 @@ void syncLayer(Crawler* c, Crawler::Layer& layer) int from = m->getPCU()->Sender(); while ( ! m->getPCU()->Unpacked()) { Entity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); if (c->recv(e, from)) layer.push_back(e); } @@ -162,7 +161,7 @@ struct LayerNumberer : public Crawler bool recv(Entity* v, int) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); if (t.hasNumber(v)) return false; t.setNumber(v, n); diff --git a/ma/maDBG.cc b/ma/maDBG.cc index 69f08c14c..8ed629fbe 100644 --- a/ma/maDBG.cc +++ b/ma/maDBG.cc @@ -13,7 +13,6 @@ #include "maRefine.h" #include #include -#include #include #include #include diff --git a/ma/maExtrude.cc b/ma/maExtrude.cc index acaade12c..13b4088b5 100644 --- a/ma/maExtrude.cc +++ b/ma/maExtrude.cc @@ -1,7 +1,6 @@ #include "maExtrude.h" #include "maCrawler.h" #include -#include #include #include diff --git a/ma/maLayer.cc b/ma/maLayer.cc index 5c2bd1191..23636699c 100644 --- a/ma/maLayer.cc +++ b/ma/maLayer.cc @@ -1,4 +1,3 @@ -#include #include "maLayer.h" #include "maAdapt.h" #include "maRefine.h" diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index 595a84312..4abd2e498 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -1,4 +1,3 @@ -#include #include "maMesh.h" #include "maAdapt.h" #include "maLayer.h" diff --git a/ma/maLayerRefine.cc b/ma/maLayerRefine.cc index 004e13463..2d2b3fe09 100644 --- a/ma/maLayerRefine.cc +++ b/ma/maLayerRefine.cc @@ -1,4 +1,3 @@ -#include #include "maCrawler.h" #include "maRefine.h" #include "maLayer.h" diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index 09840318f..c3a3dbd9a 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -1,4 +1,3 @@ -#include #include "maCrawler.h" #include "maLayer.h" #include "maSnap.h" diff --git a/ma/maMatchedCollapse.cc b/ma/maMatchedCollapse.cc index bb912eac5..c0f4091a9 100644 --- a/ma/maMatchedCollapse.cc +++ b/ma/maMatchedCollapse.cc @@ -10,7 +10,6 @@ #include "maMatchedCollapse.h" #include "maAdapt.h" #include -#include #include diff --git a/ma/maMatchedSnapper.cc b/ma/maMatchedSnapper.cc index 3c801f095..c5e87feb1 100644 --- a/ma/maMatchedSnapper.cc +++ b/ma/maMatchedSnapper.cc @@ -12,7 +12,6 @@ #include "maShapeHandler.h" #include #include -#include #include namespace ma { diff --git a/ma/maMesh.cc b/ma/maMesh.cc index 108f71838..c27c1d3b1 100644 --- a/ma/maMesh.cc +++ b/ma/maMesh.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maMesh.h" #include "maTables.h" #include diff --git a/ma/maRefine.cc b/ma/maRefine.cc index eddd0180e..1d07bbaf0 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maRefine.h" #include "maTemplates.h" #include "maAdapt.h" diff --git a/ma/maShape.cc b/ma/maShape.cc index 3e59cbeb5..aae547aee 100644 --- a/ma/maShape.cc +++ b/ma/maShape.cc @@ -8,7 +8,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maShape.h" #include "maSize.h" #include "maAdapt.h" diff --git a/ma/maSize.cc b/ma/maSize.cc index 9fdf6ed30..2a403dfc1 100644 --- a/ma/maSize.cc +++ b/ma/maSize.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maSize.h" #include "maSolutionTransferHelper.h" #include "apfMatrix.h" diff --git a/ma/maSnap.cc b/ma/maSnap.cc index 73dc2981f..5cafb87a6 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maSnap.h" #include "maAdapt.h" #include "maOperator.h" diff --git a/ma/maTetrahedronize.cc b/ma/maTetrahedronize.cc index 391456df6..332209a03 100644 --- a/ma/maTetrahedronize.cc +++ b/ma/maTetrahedronize.cc @@ -1,4 +1,3 @@ -#include #include #include "maTetrahedronize.h" #include "maCrawler.h" From 98e9d715d559e62c810c96e1702c30eb5f409383 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 16:59:55 -0500 Subject: [PATCH 060/141] changing apfMDS.cc back to original, still needs work --- mds/apfMDS.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index d07a7c8a2..82d48015c 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -973,7 +973,6 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) { double t0 = pcu::Time(); - //PCU_ALWAYS_ASSERT(PCU_GetGlobal() != m->getPCU()); int self = PCU_Comm_Self(); int outputPartCount = PCU_Comm_Peers(); apf::Expand expand(inputPartCount, outputPartCount); From 6f49fb293b700a85c80cd16d9430da468d922d1c Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 17:00:07 -0500 Subject: [PATCH 061/141] pumi pcu fixes --- pumi/pumi_ghost.cc | 11 +++++------ pumi/pumi_mentity.cc | 1 - pumi/pumi_numbering.cc | 3 +-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc index dff9f02e2..0a6235b76 100644 --- a/pumi/pumi_ghost.cc +++ b/pumi/pumi_ghost.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -197,9 +196,9 @@ static void setupGhosts(pMesh m, EntityVector& received) { int from = m->getPCU()->Sender(); pMeshEnt entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); pMeshEnt sender; - PCU_COMM_UNPACK(sender); + m->getPCU()->Unpack(sender); m->addGhost(entity, from, sender); if (!m->hasTag(entity, pumi::instance()->ghosted_tag)) { @@ -579,7 +578,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); while (global_num_off_part) { @@ -700,7 +699,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); } // while global_off_part_brg } @@ -824,7 +823,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); } if (global_num_off_part) diff --git a/pumi/pumi_mentity.cc b/pumi/pumi_mentity.cc index 8a410657b..341a680ad 100644 --- a/pumi/pumi_mentity.cc +++ b/pumi/pumi_mentity.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index f1a5a04d7..40d5a6fa2 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -14,7 +14,6 @@ #include "apfNumbering.h" #include "apfNumberingClass.h" #include -#include #include //************************************ @@ -138,7 +137,7 @@ pNumbering pumi_numbering_createProcGrp ( int* out_arr = new int[m->getPCU()->Peers()]; // out[i] has local_numOwnedPartBdryEnt of process i on all processes *in = owned_node_cnt; - MPI_Allgather(in, 1, MPI_INT, out_arr, 1, MPI_INT, PCU_Get_Comm()); + MPI_Allgather(in, 1, MPI_INT, out_arr, 1, MPI_INT, m->getPCU()->GetMPIComm()); it = m->begin(dim); int nbr = 0; From 14d2e5e7dcd16b5452c226b487bfb0c237dac2f0 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 17:00:51 -0500 Subject: [PATCH 062/141] spr pcu fix --- spr/sprRecoverField.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/spr/sprRecoverField.cc b/spr/sprRecoverField.cc index ef55aa84b..38490da1f 100644 --- a/spr/sprRecoverField.cc +++ b/spr/sprRecoverField.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "spr.h" From 5f17c05c4dbc49974c9a26c93b5e2533d7ae5cc2 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 17:01:14 -0500 Subject: [PATCH 063/141] stk pcu fixes --- stk/apfExodusOutput.cc | 1 - stk/apfSTK.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/stk/apfExodusOutput.cc b/stk/apfExodusOutput.cc index f8cc560f8..637d8ff0a 100644 --- a/stk/apfExodusOutput.cc +++ b/stk/apfExodusOutput.cc @@ -10,7 +10,6 @@ #error "configuration bug" #endif #include "apfSTK.h" -#include "PCU.h" #include #include #include diff --git a/stk/apfSTK.cc b/stk/apfSTK.cc index 1182e2ca9..b89479cc7 100644 --- a/stk/apfSTK.cc +++ b/stk/apfSTK.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfAlbany.h" From bc1ce6a43c482e8f392a356b46325838dd2362bc Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 12 Feb 2024 17:01:34 -0500 Subject: [PATCH 064/141] changing repartition.cc back to original, still needs work --- test/repartition.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/repartition.cc b/test/repartition.cc index e10e7d875..4325116f0 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -105,7 +105,7 @@ int main(int argc, char** argv) if (isOriginal) m = apf::loadMdsMesh(g, meshFile); switchToAll(); - m = apf::MdsMesh(m, g, inputPartCount); + m = apf::expandMdsMesh(m, g, inputPartCount); balance(m); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); From 59d5a7ff1d1c91b10c5669f8feadb8d701acdb42 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 14 Feb 2024 10:32:22 -0500 Subject: [PATCH 065/141] zoltan pcu fixes --- omega_h/apfOmega_h.cc | 1 - zoltan/apfInterElement.cc | 3 +-- zoltan/apfZoltan.cc | 5 ++--- zoltan/apfZoltanCallbacks.cc | 3 +-- zoltan/apfZoltanMesh.cc | 1 - 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/omega_h/apfOmega_h.cc b/omega_h/apfOmega_h.cc index 1c2ada2c9..d6134c175 100644 --- a/omega_h/apfOmega_h.cc +++ b/omega_h/apfOmega_h.cc @@ -7,7 +7,6 @@ #include #include #include -#include #include #include diff --git a/zoltan/apfInterElement.cc b/zoltan/apfInterElement.cc index f69ed6f6f..7c7183e82 100644 --- a/zoltan/apfInterElement.cc +++ b/zoltan/apfInterElement.cc @@ -1,4 +1,3 @@ -#include #include "apfZoltan.h" #include #include @@ -67,7 +66,7 @@ MeshTag* tagOpposites(GlobalNumbering* gn, const char* name) packOtherGid(gn, e); m->end(it); m->getPCU()->Send(); - while (PCU_Comm_Receive()) + while (m->getPCU()->Receive()) unpackOtherGid(m, t); return t; } diff --git a/zoltan/apfZoltan.cc b/zoltan/apfZoltan.cc index 8039523d6..b90866a07 100644 --- a/zoltan/apfZoltan.cc +++ b/zoltan/apfZoltan.cc @@ -8,7 +8,6 @@ #include "apfZoltan.h" #include "apfZoltanMesh.h" #include -#include #include namespace apf { @@ -31,12 +30,12 @@ class ZoltanSplitter : public Splitter for (int i = 0; i < plan->count(); ++i) { MeshEntity* e = plan->get(i); int p = plan->sending(e); - p += PCU_Proc_Self() * multiple; + p += bridge->mesh->getPCU()->Self() * multiple; plan->send(e, p); } } double t1 = pcu::Time(); - if (!m->getPCU()->Self()) + if (!bridge->mesh->getPCU()->Self()) lion_oprint(1, "planned Zoltan split factor %d to target" " imbalance %f in %f seconds\n", multiple, tolerance, t1 - t0); return plan; diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index 6fc125436..d1522a306 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -9,7 +9,6 @@ #include "apfZoltanMesh.h" #include "apfZoltan.h" #include "apfShape.h" -#include #include #include #include @@ -345,7 +344,7 @@ void ZoltanData::setup() if ( zb->isLocal ) { sprintf(paramStr, "%d", zb->multiple); } else { - sprintf(paramStr, "%d", zb->multiple*PCU_Proc_Peers()); + sprintf(paramStr, "%d", zb->multiple*zb->mesh->getPCU()->Peers()); } Zoltan_Set_Param(ztn, "NUM_GLOBAL_PARTS", paramStr); sprintf(paramStr, "%d", zb->multiple); diff --git a/zoltan/apfZoltanMesh.cc b/zoltan/apfZoltanMesh.cc index 09df9723e..853186022 100644 --- a/zoltan/apfZoltanMesh.cc +++ b/zoltan/apfZoltanMesh.cc @@ -8,7 +8,6 @@ #include "apfZoltanMesh.h" #include "apfZoltanCallbacks.h" #include "apfZoltan.h" -#include #include namespace apf { From 98341a3e92b312854a45f8ce86c23ad7e13c4c86 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Sun, 10 Mar 2024 23:50:34 -0400 Subject: [PATCH 066/141] pumi interface fixes, replacing old PCU uses --- pumi/pumi.h | 25 ++++---- pumi/pumi_field.cc | 32 +++++----- pumi/pumi_geom.cc | 22 +++---- pumi/pumi_ghost.cc | 26 ++++----- pumi/pumi_mentity.cc | 18 +++--- pumi/pumi_mesh.cc | 129 ++++++++++++++++++++++++++++++++++------- pumi/pumi_numbering.cc | 12 ++-- pumi/pumi_sys.cc | 30 ++++------ 8 files changed, 186 insertions(+), 108 deletions(-) diff --git a/pumi/pumi.h b/pumi/pumi.h index 529d528ba..f9a1c628f 100644 --- a/pumi/pumi.h +++ b/pumi/pumi.h @@ -107,17 +107,16 @@ class pumi // 0- SYSTEM-LEVEL FUNCTIONS //************************************ //************************************ -void pumi_start(); -void pumi_finalize(bool do_mpi_finalize=true); +void pumi_finalize(pcu::PCU *PCUObj, bool do_mpi_finalize=true); -int pumi_size(); -int pumi_rank(); +int pumi_size(pcu::PCU *PCUObj); +int pumi_rank(pcu::PCU *PCUObj); -void pumi_sync(void); -void pumi_printSys(); +void pumi_sync(pcu::PCU *PCUObj); +void pumi_printSys(pcu::PCU *PCUObj); double pumi_getTime(); double pumi_getMem(); -void pumi_printTimeMem(const char* msg, double time, double memory); +void pumi_printTimeMem(const char* msg, double time, double memory, pcu::PCU *PCUObj); //************************************ //************************************ @@ -127,12 +126,12 @@ void pumi_printTimeMem(const char* msg, double time, double memory); // Geometric Model // create a model from gmi_model object -pGeom pumi_geom_load(gmi_model* gm, +pGeom pumi_geom_load(gmi_model* gm, pcu::PCU *PCUObj, const char* model_type="mesh", const char* fileName=NULL, void (*fp)(const char*)=NULL); // create a model from a file -pGeom pumi_geom_load (const char* fileName, const char* model_type="mesh", +pGeom pumi_geom_load (const char* fileName, pcu::PCU *PCUObj, const char* model_type="mesh", void (*fp)(const char*)=NULL); void pumi_geom_delete(pGeom g); void pumi_geom_freeze(pGeom g); // shall be called after modifying model entities @@ -140,7 +139,7 @@ void pumi_geom_createID(pGeom g); // generate sequential ID starting from 1 int pumi_geom_getNumEnt(pGeom g, int d); pGeomEnt pumi_geom_findEnt(pGeom g, int d, int id); -void pumi_geom_print(pGeom g, bool print_ent=false); +void pumi_geom_print(pGeom g, pcu::PCU *PCUObj, bool print_ent=false); // Geometric Entity // get geometric entity's dimension @@ -206,17 +205,17 @@ pMeshEnt pumi_mesh_createEnt(pMesh m, pGeomEnt ge, int target_topology, pMeshEnt pMeshEnt pumi_mesh_createElem(pMesh m, pGeomEnt ge, int target_topology, pMeshEnt* vertices); // load a serial mesh and no partitioning -pMesh pumi_mesh_loadSerial(pGeom g, const char* file_name, const char* mesh_type="mds"); +pMesh pumi_mesh_loadSerial(pGeom g, const char* file_name, pcu::PCU* PCUObj, const char* mesh_type="mds"); // load a mesh from a file. Do static partitioning if num_in_part==1 -pMesh pumi_mesh_load(pGeom geom, const char* fileName, int num_in_part, const char* mesh_type="mds"); +pMesh pumi_mesh_load(pGeom geom, const char* fileName, int num_in_part, pcu::PCU* PCUObj, const char* mesh_type="mds"); // load a mesh from a an existing partitioned apf mesh pMesh pumi_mesh_load(pMesh mesh); // load a serial mesh on all processes and set up comm links and ptn classification // note that the default owning PID is 0 -pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stich_link=true); +pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool stich_link=true); // delete mesh void pumi_mesh_delete(pMesh m); diff --git a/pumi/pumi_field.cc b/pumi/pumi_field.cc index 6ca417eb4..74c520126 100644 --- a/pumi/pumi_field.cc +++ b/pumi/pumi_field.cc @@ -13,7 +13,7 @@ #include "apfFieldData.h" #include "apfNumbering.h" #include -#include +#include #include #include // for malloc and free @@ -205,7 +205,7 @@ void pumi_field_multiply(pField f1, double d, pField f2) void pumi_field_print(pField f) //******************************************************* { - pumi_sync(); + pumi_sync(getMesh(f)->getPCU()); apf::Mesh* m = getMesh(f); if (!m->findTag("global_id")) pumi_mesh_createGlobalID((pMesh)m); @@ -228,20 +228,20 @@ void pumi_field_print(pField f) switch (n) { case 1: { - std::cout<<"[p"<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())) std::cout<<__func__<<" failed for field " <isGhosted(e)) } -static void receiveFieldData(std::vector& fields, std::set& mismatch_fields) +static void receiveFieldData(std::vector& fields, std::set& mismatch_fields, pcu::PCU *PCUObj) { pField f; void *msg_recv; @@ -422,7 +422,7 @@ static void receiveFieldData(std::vector& fields, std::set& mism size_t msg_size; pMeshEnt e; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(PCUObj->Read(&pid_from, &msg_recv, &msg_size)) { e = *((pMeshEnt*)msg_recv); int *nf = (int*)((char*)msg_recv+sizeof(pMeshEnt)); @@ -465,7 +465,7 @@ void pumi_field_verify(pMesh m, pField f, pOwnership shr) if (!shr) shr = getSharing(m); - if (!pumi_rank()) // master + if (!pumi_rank(m->getPCU())) // master { lion_oprint(1," - verifying fields: "); for (size_t nf = 0; nf < fields.size(); ++nf) @@ -491,7 +491,7 @@ void pumi_field_verify(pMesh m, pField f, pOwnership shr) sendFieldData(m, e, fields[nf], nf, shr); m->end(it); m->getPCU()->Send(); - receiveFieldData(fields,mismatch_fields); + receiveFieldData(fields,mismatch_fields,m->getPCU()); } } int global_size = m->getPCU()->Max((int)mismatch_fields.size()); diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index 97c01d5b5..1c19467b5 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -12,7 +12,7 @@ #include "gmi_null.h" #include "gmi_analytic.h" #include "pumi_iter.h" -#include "PCU.h" +#include #include #include #include @@ -31,27 +31,27 @@ gEntity* gModel::getGeomEnt(int d, gmi_ent* ge) return allEntities.getGeomEnt(d, ge); } -pGeom pumi_geom_load(const char* filename, const char* model_type, void (*geom_load_fp)(const char*)) +pGeom pumi_geom_load(const char* filename, pcu::PCU *PCUObj, const char* model_type, void (*geom_load_fp)(const char*)) { if (!strcmp(model_type,"null")) { gmi_register_null(); - return pumi_geom_load(gmi_load(".null"), model_type); + return pumi_geom_load(gmi_load(".null"), PCUObj, model_type); } else if (!strcmp(model_type,"mesh")) { gmi_register_mesh(); - return pumi_geom_load(gmi_load(filename)); + return pumi_geom_load(gmi_load(filename), PCUObj); } else if (!strcmp(model_type,"analytic")) - return pumi_geom_load(gmi_make_analytic(), model_type, filename, geom_load_fp); + return pumi_geom_load(gmi_make_analytic(), PCUObj, model_type, filename, geom_load_fp); else - if (!pumi_rank()) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); + if (!pumi_rank(PCUObj)) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); return NULL; } -pGeom pumi_geom_load(gmi_model* gm, const char* model_type, +pGeom pumi_geom_load(gmi_model* gm, pcu::PCU *PCUObj, const char* model_type, const char* filename, void (*geom_load_fp)(const char*)) { double t0 = pcu::Time(); @@ -73,11 +73,11 @@ pGeom pumi_geom_load(gmi_model* gm, const char* model_type, } else { - if (!pumi_rank()) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); + if (!pumi_rank(PCUObj)) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); return NULL; } - if (!PCU_Comm_Self() && filename) + if (!PCUObj->Self() && filename) lion_oprint(1,"model %s loaded in %f seconds\n", filename, pcu::Time() - t0); return pumi::instance()->model; @@ -191,9 +191,9 @@ void pumi_giter_reset(gIter iter) iter->reset(); } -void pumi_geom_print (pGeom g, bool print_ent) +void pumi_geom_print (pGeom g, pcu::PCU *PCUObj, bool print_ent) { - if (PCU_Comm_Self()) return; + if (PCUObj->Self()) return; std::cout<<"\n=== model entity and tag info === \n"; std::cout<<"# global geom ent: v "<size(0)<<", e " <size(1)<<", f "<size(2)<<", r "<size(3)<<"\n"; diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc index 0a6235b76..d164ff79c 100644 --- a/pumi/pumi_ghost.cc +++ b/pumi/pumi_ghost.cc @@ -458,7 +458,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int layer=2; layergetPCU()); ++pid) { for (std::set::iterator off_it=off_bridge_set[layer][pid].begin(); off_it!=off_bridge_set[layer][pid].end(); ++off_it) @@ -484,7 +484,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, } // clean up for (int i=0; igetPCU());++j) off_bridge_set[i][j].clear(); m->getPCU()->Send(); @@ -576,7 +576,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, // the below runs only if num_layer>2 int global_num_off_part, local_num_off_part=0; for (int i=0; igetPCU());++j) local_num_off_part+=off_bridge_set[i][j].size(); MPI_Allreduce(&local_num_off_part, &global_num_off_part, 1,MPI_INT,MPI_SUM,m->getPCU()->GetMPIComm()); @@ -586,7 +586,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int layer=0; layergetPCU()); ++pid) { for (std::set::iterator off_it=off_bridge_set[layer][pid].begin(); off_it!=off_bridge_set[layer][pid].end(); ++off_it) @@ -612,7 +612,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, } // clean up for (int i=0; igetPCU());++j) off_bridge_set[i][j].clear(); m->getPCU()->Send(); @@ -626,7 +626,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, r_layer=r_int[0]; r_pid=r_int[1]; - PCU_ALWAYS_ASSERT(r_layer<=num_layer && r_pidgetPCU())); off_bridge_marker[r].insert(r_pid); apf::Adjacent ghost_cands; @@ -697,7 +697,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, } // while (PCU_Comm_Read) local_num_off_part=0; for (int i=0; igetPCU());++j) local_num_off_part+=off_bridge_set[i][j].size(); MPI_Allreduce(&local_num_off_part, &global_num_off_part, 1,MPI_INT,MPI_SUM,m->getPCU()->GetMPIComm()); } // while global_off_part_brg @@ -710,7 +710,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, { if (m->getPCU()->Peers()==1 || num_layer==0) return; - int dummy=1, mesh_dim=m->getDimension(), self = pumi_rank();; + int dummy=1, mesh_dim=m->getDimension(), self = pumi_rank(m->getPCU());; // brid/ghost dim check if (brg_dim>=ghost_dim || 0>brg_dim || brg_dim>=mesh_dim || @@ -738,7 +738,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, std::set** off_bridge_set=new std::set*[num_layer+1]; for (int i=0; i[pumi_size()]; + off_bridge_set[i]=new std::set[pumi_size(m->getPCU())]; apf::MeshIterator* it = m->begin(brg_dim); while ((brg_ent = m->iterate(it))) @@ -821,7 +821,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, if (num_layer>=2) { for (int i=0; igetPCU());++j) local_num_off_part+=off_bridge_set[i][j].size(); MPI_Allreduce(&local_num_off_part, &global_num_off_part, 1,MPI_INT,MPI_SUM,m->getPCU()->GetMPIComm()); } @@ -832,7 +832,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, // clean up m->destroyTag(tag); for (int i=0; igetPCU());++j) off_bridge_set[i][j].clear(); for (int i=0; i&) +void pumi_ghost_getInfo (pMesh m, std::vector&) // ********************************************************* { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: not supported\n"; } diff --git a/pumi/pumi_mentity.cc b/pumi/pumi_mentity.cc index 341a680ad..b029273d8 100644 --- a/pumi/pumi_mentity.cc +++ b/pumi/pumi_mentity.cc @@ -34,7 +34,7 @@ int pumi_ment_getNumAdj(pMeshEnt e, int target_dim) int ent_dim= apf::getDimension(pumi::instance()->mesh, e); if (ent_dim==target_dim) { - if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<": invalid target dimension "<mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<": invalid target dimension "<mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<": invalid bridge/target dimension \n"; return; // error } @@ -178,7 +178,7 @@ int pumi_ment_getOwnPID(pMeshEnt e, pOwnership o) pMeshEnt pumi_ment_getOwnEnt(pMeshEnt e, pOwnership o) { int own_partid = pumi_ment_getOwnPID(e, o); - if (own_partid==pumi_rank()) return e; + if (own_partid==pumi_rank(pumi::instance()->mesh->getPCU())) return e; if (pumi::instance()->mesh->isShared(e)) { @@ -195,14 +195,14 @@ pMeshEnt pumi_ment_getOwnEnt(pMeshEnt e, pOwnership o) bool pumi_ment_isOwned(pMeshEnt e, pOwnership o) { if (!o) - return (pumi_ment_getOwnPID(e)==pumi_rank()); + return (pumi_ment_getOwnPID(e)==pumi_rank(pumi::instance()->mesh->getPCU())); return o->isOwned(e); } bool pumi_ment_isOn(pMeshEnt e, int partID) { // FIXME: include ghost copy - if (partID==pumi_rank()) return true; + if (partID==pumi_rank(pumi::instance()->mesh->getPCU())) return true; apf::Copies remotes; pumi::instance()->mesh->getRemotes(e,remotes); APF_ITERATE(Copies,remotes,rit) @@ -250,22 +250,22 @@ pMeshEnt pumi_ment_getRmt(pMeshEnt& e, int pid) void pumi_ment_setRmt(pMeshEnt, int, pMeshEnt) { - if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } void pumi_ment_deleteRmt (pMeshEnt, int) { - if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } void pumi_ment_cleanRmt (pMeshEnt) { - if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } void pumi_ment_setPtnTopology (pMeshEnt) { - if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } int pumi_ment_getGlobalID(pMeshEnt e) diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 47737d87f..728eb619e 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -178,20 +178,20 @@ apf::Migration* getPlan(apf::Mesh* m, int num_target_part) return plan; } -void split_comm(int num_out_comm) +void split_comm(int num_out_comm, pcu::PCU &PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj.Self(); int group_id = self % num_out_comm; int in_group_rank = self / num_out_comm; MPI_Comm groupComm; - MPI_Comm_split(PCU_Get_Comm(), group_id, in_group_rank, &groupComm); - PCU_Switch_Comm(groupComm); + MPI_Comm_split(PCUObj.GetMPIComm(), group_id, in_group_rank, &groupComm); + PCUObj.SwitchMPIComm(groupComm); } -void merge_comm(MPI_Comm oldComm) +void merge_comm(MPI_Comm oldComm, pcu::PCU &PCUObj) { - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(oldComm); + MPI_Comm prevComm = PCUObj.GetMPIComm(); + PCUObj.SwitchMPIComm(oldComm); MPI_Comm_free(&prevComm); } @@ -202,6 +202,7 @@ pGeom pumi_mesh_getGeom(pMesh) } // load a serial mesh on master process then distribute as per the distribution object +/* pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, const char* mesh_type) { if (strcmp(mesh_type,"mds")) @@ -217,7 +218,28 @@ pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, const char* mesh_type) if (isMaster) m = apf::loadMdsMesh(g->getGmi(), filename); merge_comm(prevComm); - pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1); + pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1, m->getPCU()); + return pumi::instance()->mesh; +} +*/ + + +pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, pcu::PCU *PCUObj, const char* mesh_type) +{ + if (strcmp(mesh_type,"mds")) + { + if (!PCUObj->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<GetMPIComm(); + int num_target_part = PCUObj->Peers(); + bool isMaster = ((PCUObj->Self() % num_target_part) == 0); + pMesh m = 0; + split_comm(num_target_part, *PCUObj); + if (isMaster) + m = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + merge_comm(prevComm, *PCUObj); + pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1, m->getPCU()); return pumi::instance()->mesh; } @@ -229,6 +251,11 @@ pMesh pumi_mesh_load(pMesh m) } + + + + +/* pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* mesh_type) { if (strcmp(mesh_type,"mds")) @@ -256,6 +283,36 @@ pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* pumi_mesh_print(pumi::instance()->mesh); return pumi::instance()->mesh; } +*/ + +pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, pcu::PCU *PCUObj, const char* mesh_type) +{ + if (strcmp(mesh_type,"mds")) + { + if (!PCUObj->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<1) // do static partitioning + { + MPI_Comm prevComm = PCUObj->GetMPIComm(); + int num_target_part = PCUObj->Peers()/num_in_part; + bool isMaster = ((PCUObj->Self() % num_target_part) == 0); + pMesh m = 0; + apf::Migration* plan = 0; + split_comm(num_target_part, *PCUObj); + if (isMaster) { + m = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + plan = getPlan(m, num_target_part); + } + merge_comm(prevComm, *PCUObj); + pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part, PCUObj); + } + else + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + pumi_mesh_print(pumi::instance()->mesh); + return pumi::instance()->mesh; +} + @@ -279,6 +336,7 @@ void send_entities(pMesh mesh, int dim) #include "apfMDS.h" #include "apfPM.h" +/* pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) { if (pumi_size()==1) @@ -304,6 +362,33 @@ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) return pumi::instance()->mesh; } +*/ + +pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool stitch_link) +{ + if (pumi_size(PCUObj)==1) + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + else + { + double t0 = pcu::Time(); + MPI_Comm prevComm = PCUObj->GetMPIComm(); + int num_target_part = PCUObj->Peers(); + split_comm(num_target_part, *PCUObj); + // no pmodel & remote links setup + pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename); + merge_comm(prevComm, *PCUObj); + if (!PCUObj->Self()) + lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); + } + + if (pumi_size(PCUObj)>1 && stitch_link) + { + stitchMesh(pumi::instance()->mesh); + pumi::instance()->mesh->acceptChanges(); + } + + return pumi::instance()->mesh; +} void pumi_mesh_migrate(pMesh m, Migration* plan) { @@ -345,9 +430,9 @@ void pumi_mesh_setCount(pMesh m, pOwnership o) pumi::instance()->num_own_ent[dim] = n; } } - MPI_Allreduce(pumi::instance()->num_own_ent, pumi::instance()->num_global_ent, 4, MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_Allreduce(pumi::instance()->num_own_ent, pumi::instance()->num_global_ent, 4, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); #ifdef DEBUG - if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" end\n"; + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" end\n"; #endif } @@ -416,7 +501,7 @@ void pumi_mesh_print (pMesh m, bool print_ent) local_entity_count[i]=own_entity_count[i]=0; pMeshEnt e; - int self = pumi_rank(); + int self = pumi_rank(m->getPCU()); for (int d=0; d<4;++d) { @@ -425,7 +510,7 @@ void pumi_mesh_print (pMesh m, bool print_ent) while ((e = m->iterate(it))) { if (m->getOwner(e)==self) - ++own_entity_count[4*pumi_rank()+d]; + ++own_entity_count[4*pumi_rank(m->getPCU())+d]; } m->end(it); } @@ -434,10 +519,10 @@ void pumi_mesh_print (pMesh m, bool print_ent) int* global_own_entity_count = new int[4*m->getPCU()->Peers()]; MPI_Allreduce(local_entity_count, global_local_entity_count, 4*m->getPCU()->Peers(), - MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); MPI_Allreduce(own_entity_count, global_own_entity_count, 4*m->getPCU()->Peers(), - MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); if (!m->getPCU()->Self()) @@ -650,7 +735,7 @@ void pumi_ownership_verify(pMesh m, pOwnership o) own_copy = pumi_ment_getOwnEnt(e,o); if (!own_copy) { - std::cout<<"[ERROR] ("<getPCU())<<") "<<__func__<<": pumi_ment_getOwnEnt(dim "<getPCU())<<") "<<__func__<<": pumi_ment_getOwnEnt(dim "<getPCU())<<") "<<__func__<<": pumi_ment_getRmt(dim "<getPCU())<<") "<<__func__<<": pumi_ment_getRmt and pumi_ment_getOwnEnt mismatch for e(dim "<getPCU())) std::cout<<__func__<<": ownership is valid\n"; } Distribution::Distribution(pMesh mesh) @@ -810,7 +895,7 @@ static void distr_getAffected (pMesh m, Distribution* plan, EntityVector affecte while (m->getPCU()->Receive()) { pMeshEnt entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); if ( !m->hasTag(entity,tag)) { m->setIntTag(entity,tag,&dummy); @@ -881,7 +966,7 @@ static void distr_updateResidences(pMesh m, while(m->getPCU()->Receive()) { pMeshEnt entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); Parts current; m->getResidence(entity,current); Parts incoming; diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index 40d5a6fa2..e0b07c447 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -26,7 +26,7 @@ pNumbering pumi_numbering_create pNumbering n = m->findNumbering(name); if (n) { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""< #include "pumi.h" #include #include @@ -18,36 +17,31 @@ //************************************ //************************************ -void pumi_start() +void pumi_finalize(pcu::PCU *PCUObj, bool) { - PCU_Comm_Init(); + delete PCUObj; } -void pumi_finalize(bool) +int pumi_size(pcu::PCU *PCUObj) { - PCU_Comm_Free(); + return PCUObj->Peers(); } -int pumi_size() +int pumi_rank(pcu::PCU *PCUObj) { - return PCU_Comm_Peers(); + return PCUObj->Self(); } -int pumi_rank() +void pumi_sync(pcu::PCU *PCUObj) { - return PCU_Comm_Self(); -} - -void pumi_sync(void) -{ - MPI_Barrier(PCU_Get_Comm()); + MPI_Barrier(PCUObj->GetMPIComm()); } #include #include -void pumi_printSys() +void pumi_printSys(pcu::PCU *PCUObj) { - if (PCU_Comm_Self()) return; + if (PCUObj->Self()) return; struct utsname u; if (uname(&u) == 0) lion_oprint(1,"[%s] %s %s %s %s %s\n\n", @@ -67,9 +61,9 @@ double pumi_getMem() return pcu::GetMem(); } -void pumi_printTimeMem(const char* msg, double time, double memory) +void pumi_printTimeMem(const char* msg, double time, double memory, pcu::PCU *PCUObj) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) { lion_oprint(1,"%-20s %6.3f sec %7.3f MB \n", msg, time, memory); fflush(stdout); From a0c53fc7420c2382a060503703b118b92555852e Mon Sep 17 00:00:00 2001 From: flagdanger Date: Sun, 10 Mar 2024 23:56:11 -0400 Subject: [PATCH 067/141] files in test folder changed to use new pumi - pcu interface --- test/constructThenGhost.cc | 14 ++++---- test/pumi.cc | 73 +++++++++++++++++++------------------- test/pumiLoadMesh.cc | 8 ++--- test/xgc_split.cc | 20 +++++------ 4 files changed, 58 insertions(+), 57 deletions(-) diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index e90e41b34..34fae29fb 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -14,7 +14,7 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -24,7 +24,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], PCUObj); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -32,7 +32,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false); + m = apf::makeEmptyMdsMesh(model, dim, false, PCUObj); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; @@ -43,10 +43,10 @@ int main(int argc, char** argv) outMap.clear(); m->verify(); - if (!pumi_rank()) printf("model/mesh converted to pumi instance\n"); + if (!pumi_rank(PCUObj)) printf("model/mesh converted to pumi instance\n"); //create the pumi instance to use pumi api's - pGeom g=pumi_geom_load(model); + pGeom g = pumi_geom_load(model, PCUObj); pMesh pm = pumi_mesh_load(m); pumi_mesh_verify(pm); @@ -91,6 +91,6 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(pm); - PCU_Comm_Free(); + delete PCUObj; MPI_Finalize(); } diff --git a/test/pumi.cc b/test/pumi.cc index 8bcc88912..c407f612a 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -37,10 +37,10 @@ struct testOwnership : public Ownership pOwnership o; }; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc < 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -87,29 +87,30 @@ int main(int argc, char** argv) //********************************************************* { MPI_Init(&argc,&argv); - pumi_start(); + //pumi_start(); + pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - pumi_printSys(); + pumi_printSys(PCUObj); #if 0 int i, processid = getpid(); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) { - std::cout<<"Proc "<> pid "<Self()<<">> pid "<>i; } else - std::cout<<"Proc "<> pid "<Self()<<">> pid "<begin(2); gent_it!=g->end(2);++gent_it) @@ -136,16 +137,16 @@ int main(int argc, char** argv) } // load mesh per process group - PCU_ALWAYS_ASSERT(pumi_size()%num_in_part==0); + PCU_ALWAYS_ASSERT(pumi_size(PCUObj)%num_in_part==0); double begin_mem = pumi_getMem(), begin_time=pumi_getTime(); pMesh m=NULL; if (do_distr) - m = pumi_mesh_loadSerial(g, meshFile); + m = pumi_mesh_loadSerial(g, meshFile, PCUObj); else { - m = pumi_mesh_load(g, meshFile, num_in_part); // static partitioning if num_in_part=1 + m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj); // static partitioning if num_in_part=1 if (num_in_part==1) pumi_mesh_write(m,"mesh.smb"); } @@ -180,7 +181,7 @@ int main(int argc, char** argv) m->end(it); pumi_mesh_distribute(m, plan); - if (!pumi_rank()) std::cout<<"\n[test_pumi] writing mesh in vtk\n"; + if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] writing mesh in vtk\n"; // write mesh in .smb pumi_mesh_write(m,outFile); @@ -194,18 +195,18 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(m); - g = pumi_geom_load(modelFile); - if (num_in_part==1 && pumi_size()>1) - m = pumi_mesh_load(g, "mesh.smb", pumi_size()); + g = pumi_geom_load(modelFile, PCUObj); + if (num_in_part==1 && pumi_size(PCUObj)>1) + m = pumi_mesh_load(g, "mesh.smb", pumi_size(PCUObj), PCUObj); else - m = pumi_mesh_load(g, meshFile, num_in_part); - if (!pumi_rank()) std::cout<<"\n[test_pumi] delete and reload mesh\n"; + m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj); + if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] delete and reload mesh\n"; pOwnership o=new testOwnership(m); pumi_ownership_verify(m, o); delete o; - if (!pumi_rank()) std::cout<<"\n[test_pumi] clean loaded tags from the mesh file\n"; + if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] clean loaded tags from the mesh file\n"; std::vector tag_vec; for (size_t n = 0; n::iterator fit=fields.begin(); fit!=fields.end(); ++fit) pumi_field_freeze(*fit); - if (!pumi_rank()) std::cout<<"\n[test_pumi] "<::iterator fit=fields.begin(); fit!=fields.end(); ++fit) pumi_field_delete(*fit); - if (!pumi_rank()) std::cout<<"\n[test_pumi] field and numbering deleted\n"; + if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] field and numbering deleted\n"; pumi_mesh_verify(m); TEST_MESH_TAG(m); @@ -247,9 +248,9 @@ int main(int argc, char** argv) pumi_mesh_delete(m); // print elapsed time and increased heap memory - pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem); + pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem, PCUObj); - pumi_finalize(); + pumi_finalize(PCUObj); MPI_Finalize(); } @@ -291,13 +292,13 @@ void TEST_MESH(pMesh m) for (pCopyIter it = copies.begin(); it != copies.end(); ++it) - PCU_ALWAYS_ASSERT(it->first!=pumi_rank()); + PCU_ALWAYS_ASSERT(it->first!=pumi_rank(m->getPCU())); // check the entity is not ghost or ghosted PCU_ALWAYS_ASSERT(!pumi_ment_isGhost(e) && !pumi_ment_isGhosted(e)); } m->end(mit); - if (!pumi_rank()) std::cout<<"\n[test_pumi] testing mesh/entity apis\n"; + if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] testing mesh/entity apis\n"; } #include @@ -635,7 +636,7 @@ void TEST_NEW_MESH(pMesh m) PCU_ALWAYS_ASSERT(pumi_shape_getNumNode(pumi_mesh_getShape(m), 1)==1); // create an empty mesh - pGeom new_g = pumi_geom_load("", "null"); + pGeom new_g = pumi_geom_load("", m->getPCU(), "null"); pMesh new_m = pumi_mesh_create(new_g, 2); double xyz[3]; @@ -674,7 +675,7 @@ void TEST_NEW_MESH(pMesh m) // pumi_mesh_freeze(new_m); - if (!pumi_rank()) std::cout<<"\n[test_pumi] new mesh constructed (#v " + if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] new mesh constructed (#v " <iterate(it))) { - for (int i=0; igetPCU())/2; ++i) { - int pid = rand()%pumi_size(); + int pid = rand()%pumi_size(m->getPCU()); plan->send(e, pid); } ++count; @@ -790,7 +791,7 @@ void TEST_GHOSTING(pMesh m) int total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - if (!pumi_rank()) std::cout<<"\n[test_pumi] element-wise pumi_ghost_create: #ghost increase="<getPCU())) std::cout<<"\n[test_pumi] element-wise pumi_ghost_create: #ghost increase="<begin(0); @@ -800,7 +801,7 @@ void TEST_GHOSTING(pMesh m) if (pumi_ment_isGhost(e)) { ++num_ghost_vtx; - PCU_ALWAYS_ASSERT(pumi_ment_getOwnPID(e)!=pumi_rank()); + PCU_ALWAYS_ASSERT(pumi_ment_getOwnPID(e)!=pumi_rank(m->getPCU())); } } m->end(mit); @@ -821,7 +822,7 @@ void TEST_GHOSTING(pMesh m) pumi_ghost_createLayer (m, brg_dim, mesh_dim, num_layer, include_copy); total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - if (!pumi_rank()) std::cout<<"\n[test_pumi] layer-wise pumi_ghost_createLayer (bd "<getPCU())) std::cout<<"\n[test_pumi] layer-wise pumi_ghost_createLayer (bd "<getPCU()->GetMPIComm()); - if (!pumi_rank()) + if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] accumulative pumi_ghost_createLayer (bd "<getPCU())<<") ERROR dim "< \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -33,11 +33,11 @@ Migration* get_xgc_plan(pGeom g, pMesh m) { int dim = pumi_mesh_getDim(m); Migration* plan = new Migration(m); - if (!pumi_rank()) return plan; + if (!pumi_rank(m->getPCU())) return plan; pMeshEnt e; int num_gface = pumi_geom_getNumEnt(g, dim); - PCU_ALWAYS_ASSERT(num_gface==pumi_size()); + PCU_ALWAYS_ASSERT(num_gface==pumi_size(m->getPCU())); int gface_id; int dest_pid; pMeshIter it = m->begin(2); // face @@ -55,22 +55,22 @@ Migration* get_xgc_plan(pGeom g, pMesh m) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - pumi_start(); + pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); - getConfig(argc,argv); + getConfig(argc,argv,PCUObj); - pGeom g = pumi_geom_load(modelFile); + pGeom g = pumi_geom_load(modelFile, PCUObj); pMesh m; if (serial) { - m = pumi_mesh_loadSerial(g, meshFile); + m = pumi_mesh_loadSerial(g, meshFile, PCUObj); // split a serial mesh based on model ID Migration* plan = get_xgc_plan(g, m); pumi_mesh_migrate(m, plan); pumi_mesh_write(m, outFile); } else - m = pumi_mesh_load(g, meshFile, pumi_size()); + m = pumi_mesh_load(g, meshFile, pumi_size(PCUObj), PCUObj); // write to vtk char without_extension[256]; @@ -93,7 +93,7 @@ int main(int argc, char** argv) pumi_mesh_delete(m); - pumi_finalize(); + pumi_finalize(PCUObj); MPI_Finalize(); } From 32ab95b5d643f7bdbc0d419cdfac8f1edbaefddf Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 12 Mar 2024 13:29:20 -0400 Subject: [PATCH 068/141] made default PCU arguments for Mesh functions and fixed old pcu issues, there are still some duplicate functions: loadMdsMesh, repeat, and expand that can be removed or made to take a default argument --- apf/apfCGNS.cc | 2 - apf/apfMesh.cc | 36 ++++++++------- apf/apfMesh.h | 8 ++-- apf/apfMesh2.cc | 40 ++++++++--------- apf/apfVerify.cc | 84 +++++++++++++++++----------------- mds/apfMDS.cc | 114 ++++++++++++++++++++++++++++++++++++++++++++--- mds/apfMDS.h | 12 ++++- 7 files changed, 206 insertions(+), 90 deletions(-) diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 69aab3246..33037d47c 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfMesh.h" #include "apfNumbering.h" #include "apfNumberingClass.h" @@ -13,7 +12,6 @@ #include "apfFieldData.h" #include #include -// #include #include #include diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 069c62c29..490806561 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -160,7 +160,7 @@ Mesh::Type const Mesh::simplexTypes[4] = ,Mesh::TET }; -void Mesh::init(FieldShape* s) +void Mesh::init(FieldShape* s, pcu::PCU *PCUObj) { coordinateField = new VectorField(); FieldBase* baseP = coordinateField; @@ -168,7 +168,13 @@ void Mesh::init(FieldShape* s) baseP->init("coordinates",this,s,data); data->init(baseP); hasFrozenFields = false; - pcu_ = pcu::PCU_GetGlobal(); + if(PCUObj != nullptr){ + pcu_ = PCUObj; + } else { + pcu_ = pcu::PCU_GetGlobal(); + PCU_ALWAYS_ASSERT_VERBOSE(pcu_ != nullptr, "PCU must be initialized\n"); + } + } Mesh::~Mesh() @@ -885,7 +891,7 @@ void printStats(Mesh* m) n[i] = countOwned(m, i); m->getPCU()->Add(n, 4); printTypes(m); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"mesh entity counts: v %ld e %ld f %ld r %ld\n", n[0], n[1], n[2], n[3]); } @@ -1027,7 +1033,7 @@ void MatchedSharing::formCountMap() mesh->getPCU()->Send(); while (mesh->getPCU()->Receive()) { size_t oc; - PCU_COMM_UNPACK(oc); + mesh->getPCU()->Unpack(oc); countMap[mesh->getPCU()->Sender()] = oc; } } @@ -1170,20 +1176,20 @@ void getAlignment(Mesh* m, MeshEntity* elem, MeshEntity* boundary, rotate = findIn(bv, nbv, ebv[0]); } -void packString(std::string s, int to) +void packString(std::string s, int to, pcu::PCU *PCUObj) { size_t len = s.length(); - PCU_COMM_PACK(to, len); - PCU_Comm_Pack(to, s.c_str(), len); + PCUObj->Pack(to, len); + PCUObj->Pack(to, s.c_str(), len); } -std::string unpackString() +std::string unpackString(pcu::PCU *PCUObj) { std::string s; size_t len; - PCU_COMM_UNPACK(len); + PCUObj->Unpack(len); s.resize(len); - PCU_Comm_Unpack((void*)s.c_str(), len); + PCUObj->Unpack((void*)s.c_str(), len); return s; } @@ -1191,7 +1197,7 @@ void packTagInfo(Mesh* m, MeshTag* t, int to) { std::string name; name = m->getTagName(t); - packString(name, to); + packString(name, to, m->getPCU()); int type; type = m->getTagType(t); m->getPCU()->Pack(to, type); @@ -1200,11 +1206,11 @@ void packTagInfo(Mesh* m, MeshTag* t, int to) m->getPCU()->Pack(to, size); } -void unpackTagInfo(std::string& name, int& type, int& size) +void unpackTagInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj) { - name = unpackString(); - PCU_COMM_UNPACK(type); - PCU_COMM_UNPACK(size); + name = unpackString(PCUObj); + PCUObj->Unpack(type); + PCUObj->Unpack(size); } char const* const dimName[4] = { diff --git a/apf/apfMesh.h b/apf/apfMesh.h index e5680c133..772c5225e 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -109,7 +109,7 @@ class Mesh \param s the field distribution of the coordinate field, apf::getLagrange(1) is a good default */ - void init(FieldShape* s); + void init(FieldShape* s, pcu::PCU *PCUObj = nullptr); /** \brief destroy the base class structures. \details this does not destroy the underlying data structure, use apf::Mesh::destroyNative for that. @@ -620,10 +620,10 @@ int getFirstType(Mesh* m, int dim); void getAlignment(Mesh* m, MeshEntity* elem, MeshEntity* boundary, int& which, bool& flip, int& rotate); -void packString(std::string s, int to); -std::string unpackString(); +void packString(std::string s, int to, pcu::PCU *PCUObj); +std::string unpackString(pcu::PCU *PCUObj); void packTagInfo(Mesh* m, MeshTag* t, int to); -void unpackTagInfo(std::string& name, int& type, int& size); +void unpackTagInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj); extern char const* const dimName[4]; diff --git a/apf/apfMesh2.cc b/apf/apfMesh2.cc index cfcd687b5..090b00ad3 100644 --- a/apf/apfMesh2.cc +++ b/apf/apfMesh2.cc @@ -163,14 +163,14 @@ static void packProposal(Mesh* m, MeshEntity* e, int to) } } -static void unpackProposal(int& t, MeshEntity*& e, Downward& da) +static void unpackProposal(int& t, MeshEntity*& e, Downward& da, pcu::PCU *PCUObj) { - PCU_COMM_UNPACK(t); - PCU_COMM_UNPACK(e); + PCUObj->Unpack(t); + PCUObj->Unpack(e); int nd; - PCU_COMM_UNPACK(nd); + PCUObj->Unpack(nd); for (int i=0; i < nd; ++i) - PCU_COMM_UNPACK(da[i]); + PCUObj->Unpack(da[i]); } void stitchMesh(Mesh2* m) @@ -195,7 +195,7 @@ void stitchMesh(Mesh2* m) while (!m->getPCU()->Unpacked()) { int t; Downward da; - unpackProposal(t, e, da); + unpackProposal(t, e, da, m->getPCU()); MeshEntity* found = findUpward(m, t, da); if (found) m->addRemote(found, from, e); @@ -209,7 +209,7 @@ static void packTagClone(Mesh2* m, MeshTag* t, int to) { std::string name; name = m->getTagName(t); - packString(name, to); + packString(name, to, m->getPCU()); int type; type = m->getTagType(t); m->getPCU()->Pack(to, type); @@ -220,11 +220,11 @@ static void packTagClone(Mesh2* m, MeshTag* t, int to) static MeshTag* unpackTagClone(Mesh2* m) { - std::string name = unpackString(); + std::string name = unpackString(m->getPCU()); int type; - PCU_COMM_UNPACK(type); + m->getPCU()->Unpack(type); int size; - PCU_COMM_UNPACK(size); + m->getPCU()->Unpack(size); if (type == apf::Mesh::DOUBLE) return m->createDoubleTag(name.c_str(), size); if (type == apf::Mesh::INT) @@ -250,7 +250,7 @@ static void packTagClones(Mesh2* m, int to) static void unpackTagClones(Mesh2* m) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (int i = 0; i < n; ++i) unpackTagClone(m); } @@ -258,24 +258,24 @@ static void unpackTagClones(Mesh2* m) static void packFieldClone(Field* f, int to) { std::string name = f->getName(); - packString(name, to); + packString(name, to, f->getMesh()->getPCU()); int valueType = f->getValueType(); f->getMesh()->getPCU()->Pack(to, valueType); int components = f->countComponents(); f->getMesh()->getPCU()->Pack(to, components); std::string shapeName = f->getShape()->getName(); - packString(shapeName, to); + packString(shapeName, to, f->getMesh()->getPCU()); /* warning! this only supports tag-stored fields */ } static Field* unpackFieldClone(Mesh2* m) { - std::string name = unpackString(); + std::string name = unpackString(m->getPCU()); int valueType; - PCU_COMM_UNPACK(valueType); + m->getPCU()->Unpack(valueType); int components; - PCU_COMM_UNPACK(components); - std::string shapeName = unpackString(); + m->getPCU()->Unpack(components); + std::string shapeName = unpackString(m->getPCU()); FieldShape* shape = getShapeByName(shapeName.c_str()); PCU_ALWAYS_ASSERT(shape); /* warning! this only supports tag-stored fields */ @@ -293,7 +293,7 @@ static void packFieldClones(Mesh2* m, int to) static void unpackFieldClones(Mesh2* m) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (int i = 0; i < n; ++i) unpackFieldClone(m); } @@ -301,12 +301,12 @@ static void unpackFieldClones(Mesh2* m) static void packMeshShape(Mesh2* m, int to) { std::string shapeName = m->getShape()->getName(); - packString(shapeName, to); + packString(shapeName, to, m->getPCU()); } static void unpackMeshShape(Mesh2* m) { - std::string shapeName = unpackString(); + std::string shapeName = unpackString(m->getPCU()); FieldShape* shape = getShapeByName(shapeName.c_str()); PCU_ALWAYS_ASSERT(shape); if (shape != m->getShape()) { diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc index 0f48755e6..0af98cd81 100644 --- a/apf/apfVerify.cc +++ b/apf/apfVerify.cc @@ -216,39 +216,41 @@ static Copies getAllCopies(Mesh* m, MeshEntity* e) return c; } -static void verifyAllCopies(Copies& a) +static void verifyAllCopies(Copies& a, pcu::PCU *PCUObj) { APF_ITERATE(Copies, a, it) { PCU_ALWAYS_ASSERT(it->first >= 0); - PCU_ALWAYS_ASSERT(it->first < PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(it->first < PCUObj->Peers()); } } static void packCopies( int to, - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n = copies.size(); - PCU_COMM_PACK(to,n); + PCUObj->Pack(to,n); APF_ITERATE(Copies,copies,it) { - PCU_COMM_PACK(to,it->first); - PCU_COMM_PACK(to,it->second); + PCUObj->Pack(to,it->first); + PCUObj->Pack(to,it->second); } } static void unpackCopies( - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (int i=0; i < n; ++i) { int part; - PCU_COMM_UNPACK(part); + PCUObj->Unpack(part); MeshEntity* remote; - PCU_COMM_UNPACK(remote); + PCUObj->Unpack(remote); PCU_ALWAYS_ASSERT(remote); copies[part]=remote; } @@ -259,22 +261,22 @@ static void sendAllCopies(Mesh* m, MeshEntity* e) Copies a; Copies r; a = getAllCopies(m, e); - verifyAllCopies(a); + verifyAllCopies(a, m->getPCU()); m->getRemotes(e, r); PCU_ALWAYS_ASSERT(!r.count(m->getPCU()->Self())); APF_ITERATE(Copies, r, it) { m->getPCU()->Pack(it->first, it->second); - packCopies(it->first, a); + packCopies(it->first, a, m->getPCU()); } } static void receiveAllCopies(Mesh* m) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Copies a; - unpackCopies(a); + unpackCopies(a, m->getPCU()); Copies b = getAllCopies(m, e); PCU_ALWAYS_ASSERT(a == b); } @@ -313,9 +315,9 @@ static void receiveGhostCopies(Mesh* m) { int from = m->getPCU()->Sender(); MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); MeshEntity* g; - PCU_COMM_UNPACK(g); + m->getPCU()->Unpack(g); PCU_ALWAYS_ASSERT(m->isGhost(e) || m->isGhosted(e)); Copies ghosts; m->getGhosts(e,ghosts); @@ -359,22 +361,22 @@ static bool hasMatch( return false; } -static void sendSelfToMatches(MeshEntity* e, Matches& matches) +static void sendSelfToMatches(MeshEntity* e, Matches& matches, pcu::PCU *PCUObj) { APF_ITERATE(Matches, matches, it) { - PCU_ALWAYS_ASSERT(!((it->peer == PCU_Comm_Self())&&(it->entity == e))); - PCU_COMM_PACK(it->peer, e); - PCU_COMM_PACK(it->peer, it->entity); + PCU_ALWAYS_ASSERT(!((it->peer == PCUObj->Self())&&(it->entity == e))); + PCUObj->Pack(it->peer, e); + PCUObj->Pack(it->peer, it->entity); } } static void receiveMatches(Mesh* m) { MeshEntity* source; - PCU_COMM_UNPACK(source); + m->getPCU()->Unpack(source); MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Matches matches; m->getMatches(e, matches); PCU_ALWAYS_ASSERT(hasMatch(matches, m->getPCU()->Sender(), source)); @@ -400,7 +402,7 @@ static void verifyMatches(Mesh* m) Matches matches; m->getMatches(e, matches); PCU_ALWAYS_ASSERT(!hasDuplicates(matches)); - sendSelfToMatches(e, matches); + sendSelfToMatches(e, matches, m->getPCU()); } m->end(it); } @@ -430,9 +432,9 @@ static bool receiveCoords(Mesh* m) MeshEntity* e; Vector3 ox; Vector3 op; - PCU_COMM_UNPACK(e); - PCU_COMM_UNPACK(ox); - PCU_COMM_UNPACK(op); + m->getPCU()->Unpack(e); + m->getPCU()->Unpack(ox); + m->getPCU()->Unpack(op); Vector3 x; Vector3 p(0,0,0); m->getPoint(e, 0, x); @@ -509,12 +511,12 @@ static void sendAlignment(Mesh* m, MeshEntity* e) static void receiveAlignment(Mesh* m) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int d = getDimension(m, e); Downward down; int nd = m->getDownward(e, d - 1, down); for (int i = 0; i < nd; ++i) { - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); PCU_ALWAYS_ASSERT(down[i] == e); } } @@ -535,23 +537,23 @@ static void verifyAlignment(Mesh* m) } } -void packFieldInfo(Field* f, int to) +void packFieldInfo(Field* f, int to, pcu::PCU *PCUObj) { std::string name; name = getName(f); - packString(name, to); + packString(name, to, PCUObj); int type, size; type = getValueType(f); - PCU_COMM_PACK(to, type); + PCUObj->Pack(to, type); size = countComponents(f); - PCU_COMM_PACK(to, size); + PCUObj->Pack(to, size); } -void unpackFieldInfo(std::string& name, int& type, int& size) +void unpackFieldInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj) { - name = unpackString(); - PCU_COMM_UNPACK(type); - PCU_COMM_UNPACK(size); + name = unpackString(PCUObj); + PCUObj->Unpack(type); + PCUObj->Unpack(size); } static void verifyFields(Mesh* m) @@ -567,19 +569,19 @@ static void verifyFields(Mesh* m) if (self) { m->getPCU()->Pack(self - 1, n); for (int i = 0; i < n; ++i) - packFieldInfo(fields[i], self - 1); + packFieldInfo(fields[i], self - 1, m->getPCU()); } m->getPCU()->Send(); while (m->getPCU()->Receive()) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); PCU_ALWAYS_ASSERT(fields.size() == (size_t)n); for (int i = 0; i < n; ++i) { std::string name; int type; int size; - unpackTagInfo(name, type, size); + unpackTagInfo(name, type, size, m->getPCU()); PCU_ALWAYS_ASSERT(name == getName(fields[i])); PCU_ALWAYS_ASSERT(type == getValueType(fields[i])); PCU_ALWAYS_ASSERT(size == countComponents(fields[i])); @@ -757,13 +759,13 @@ static void verifyTags(Mesh* m) m->getPCU()->Send(); while (m->getPCU()->Receive()) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); PCU_ALWAYS_ASSERT(tags.getSize() == (size_t)n); for (int i = 0; i < n; ++i) { std::string name; int type; int size; - unpackTagInfo(name, type, size); + unpackTagInfo(name, type, size, m->getPCU()); PCU_ALWAYS_ASSERT(name == m->getTagName(tags[i])); PCU_ALWAYS_ASSERT(type == m->getTagType(tags[i])); PCU_ALWAYS_ASSERT(size == m->getTagSize(tags[i])); diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 82d48015c..7ebfaa27e 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -10,6 +10,7 @@ #include #include +#include #include #include "apfMDS.h" #include "mds_apf.h" @@ -168,9 +169,13 @@ class MeshMDS : public Mesh2 isMatched = false; ownsModel = false; } - MeshMDS(gmi_model* m, int d, bool isMatched_) + MeshMDS(gmi_model* m, int d, bool isMatched_, pcu::PCU *PCUObj = nullptr) { - init(apf::getLagrange(1)); + if(PCUObj != nullptr){ + init(apf::getLagrange(1), PCUObj); + } else { + init(apf::getLagrange(1)); + } mds_id cap[MDS_TYPES] = {}; mesh = mds_apf_create(m, d, cap); isMatched = isMatched_; @@ -196,9 +201,13 @@ class MeshMDS : public Mesh2 apf::convert(from, this, nodes, elems, copy_data); } - MeshMDS(gmi_model* m, const char* pathname) + MeshMDS(gmi_model* m, const char* pathname, pcu::PCU *PCUObj = nullptr) { - init(apf::getLagrange(1)); + if(PCUObj != nullptr){ + init(apf::getLagrange(1), PCUObj); + } else { + init(apf::getLagrange(1)); + } mesh = mds_read_smb2(this->getPCU()->GetCHandle(), m, pathname, 0, this); isMatched = this->getPCU()->Or(!mds_net_empty(&mesh->matches)); ownsModel = true; @@ -778,9 +787,14 @@ class MeshMDS : public Mesh2 bool ownsModel; }; -Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched) +Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj) { - Mesh2* m = new MeshMDS(model, dim, isMatched); + Mesh2* m; + if(PCUObj != nullptr){ + m = new MeshMDS(model, dim, isMatched, PCUObj); + } else { + m = new MeshMDS(model, dim, isMatched); + } initResidence(m, dim); return m; } @@ -928,6 +942,22 @@ Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile) return m; } +Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) +{ + double t0 = pcu::Time(); + Mesh2* m = new MeshMDS(model, meshfile, PCUObj); + initResidence(m, m->getDimension()); + stitchMesh(m); + m->acceptChanges(); + + if (!m->getPCU()->Self()) + lion_oprint(1,"mesh %s loaded in %f seconds\n", meshfile, pcu::Time() - t0); + printStats(m); + warnAboutEmptyParts(m); + return m; +} + + Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile) { double t0 = pcu::Time(); @@ -943,6 +973,18 @@ Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile) return m; } + +Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj) +{ + double t0 = pcu::Time(); + static gmi_model* model; + model = gmi_load(modelfile); + if (!PCUObj->Self()) + lion_oprint(1,"model %s loaded in %f seconds\n", modelfile, pcu::Time() - t0); + + return loadMdsMesh(model, meshfile, PCUObj); +} + Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile) { double t0 = pcu::Time(); @@ -954,6 +996,8 @@ Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile) return loadMdsMesh(model, meshfile); } + + void reorderMdsMesh(Mesh2* mesh, MeshTag* t) { double t0 = pcu::Time(); @@ -970,6 +1014,46 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) lion_oprint(1,"mesh reordered in %f seconds\n", pcu::Time()-t0); } + +Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU) +{ + double t0 = pcu::Time(); + int self = expandedPCU->Self(); + //PCU_ALWAYS_ASSERT_VERBOSE(self == expandedPCU->Self(), "pcu_comm_self doesn't equal pcuobj->self\n"); + int outputPartCount = expandedPCU->Peers(); + apf::Expand expand(inputPartCount, outputPartCount); + apf::Contract contract(inputPartCount, outputPartCount); + bool isOriginal = contract.isValid(self); + int dim; + bool isMatched; + expandedPCU->Begin(); + if (isOriginal) { + PCU_ALWAYS_ASSERT(m != 0); + dim = m->getDimension(); + isMatched = m->hasMatching(); + for (int i = self + 1; i < outputPartCount && !contract.isValid(i); ++i) { + expandedPCU->Pack(i, dim); + expandedPCU->Pack(i, isMatched); + packDataClone(m, i); + } + } + expandedPCU->Send(); + while (expandedPCU->Receive()) { + expandedPCU->Unpack(dim); + expandedPCU->Unpack(isMatched); + m = makeEmptyMdsMesh(g, dim, isMatched, expandedPCU); + unpackDataClone(m); + } + PCU_ALWAYS_ASSERT(m != 0); + apf::remapPartition(m, expand); + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) + lion_oprint(1,"mesh expanded from %d to %d parts in %f seconds\n", + inputPartCount, outputPartCount, t1 - t0); + return m; +} + + Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) { double t0 = pcu::Time(); @@ -1024,6 +1108,24 @@ Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, return m; } +Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, + int factor, pcu::PCU *PCUObj) +{ + //PCU_ALWAYS_ASSERT_VERBOSE(PCU_Comm_Self() == PCUObj->Self(), "repeartMdsMesh assert failed\n"); + m = expandMdsMesh(m, g, PCUObj->Peers() / factor, PCUObj); + double t0 = pcu::Time(); + if (PCUObj->Self() % factor != 0) + plan = new apf::Migration(m, m->findTag("apf_migrate")); + m->migrate(plan); + double t1 = pcu::Time(); + if (!PCUObj->Self()) + lion_oprint(1,"mesh migrated from %d to %d in %f seconds\n", + PCUObj->Peers() / factor, + PCUObj->Peers(), + t1 - t0); + return m; +} + bool alignMdsMatches(Mesh2* in) { if (!in->hasMatching()) diff --git a/mds/apfMDS.h b/mds/apfMDS.h index 3d0ac3410..6caea293b 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -37,6 +37,10 @@ // but trying to avoid that since it's not core functionality #include // +namespace pcu{ + class PCU; + PCU* PCU_GetGlobal(); +} struct gmi_model; namespace apf { @@ -58,7 +62,7 @@ typedef struct PCUHandle PCUHandle; \param dim the eventual mesh dimension. MDS needs to allocate arrays based on this before users add entities. \param isMatched whether or not there will be matched entities */ -Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched); +Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj = nullptr); /** \brief load an MDS mesh and model from file \param modelfile will be passed to gmi_load to get the model @@ -67,6 +71,7 @@ Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched); may also be called to enable loading of GeomSim, Parasolid, and ACIS models */ +Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj); Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile); /** \brief load an MDS mesh from files @@ -82,6 +87,7 @@ Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile); using PCU file IO functions. Calling apf::Mesh::writeNative on the resulting object will do the same in reverse. */ +Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile); // make a serial mesh on all processes - no pmodel & remote link setup @@ -111,8 +117,10 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder=false, bool copy void reorderMdsMesh(Mesh2* mesh, MeshTag* t = 0); Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor); -Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount); +Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj); +Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount); +Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU); /** \brief align the downward adjacencies of matched entities */ bool alignMdsMatches(Mesh2* in); /** \brief align the downward adjacencies of remote copies */ From 530a82ec91efd47b54e8d495f3a133e78ac75d61 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 12 Mar 2024 13:29:46 -0400 Subject: [PATCH 069/141] repartition test file rebuilt to use new pcu --- test/repartition.cc | 49 ++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/test/repartition.cc b/test/repartition.cc index 4325116f0..ca3913d64 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -17,16 +17,22 @@ const char* meshFile = 0; const char* outFile = 0; int inputPartCount = 1; +struct CreateGroupCommResult{ + bool isOriginal; + pcu::PCU *group_pcu_obj; +}; + void freeMesh(apf::Mesh* m) { m->destroyNative(); apf::destroyMesh(m); } -bool switchToOriginals() +CreateGroupCommResult createGroupComm(pcu::PCU *PCUObj) { - apf::Contract contract(inputPartCount, PCU_Comm_Peers()); + apf::Contract contract(inputPartCount, PCUObj->Peers()); int self = PCU_Comm_Self(); + PCU_ALWAYS_ASSERT(self == PCUObj->Self()); int group; int groupRank; bool isOriginal = contract.isValid(self); @@ -41,22 +47,17 @@ bool switchToOriginals() } MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); - return isOriginal; + CreateGroupCommResult result; + result.isOriginal = isOriginal; + result.group_pcu_obj = new pcu::PCU(groupComm); + return result; } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: mpirun -n %s" " \n" "Increase the part count of inMesh from inPartCount to outPartCount.\n" @@ -70,7 +71,7 @@ void getConfig(int argc, char** argv) inputPartCount = atoi(argv[2]); meshFile = argv[3]; outFile = argv[4]; - PCU_ALWAYS_ASSERT(inputPartCount <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(inputPartCount <= PCUObj->Peers()); } void balance(apf::Mesh2* m) @@ -92,25 +93,27 @@ void balance(apf::Mesh2* m) } + int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + pcu::PCU *expanded_pcu_obj = new pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,expanded_pcu_obj); gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = 0; - bool isOriginal = switchToOriginals(); - if (isOriginal) - m = apf::loadMdsMesh(g, meshFile); - switchToAll(); - m = apf::expandMdsMesh(m, g, inputPartCount); + CreateGroupCommResult result = createGroupComm(expanded_pcu_obj); + + if (result.isOriginal) + m = apf::loadMdsMesh(g, meshFile, result.group_pcu_obj); + + m = apf::expandMdsMesh(m, g, inputPartCount, expanded_pcu_obj); balance(m); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); - PCU_Comm_Free(); + MPI_Finalize(); } From fda3f51bc4a4f5c6eba9ba7cc4ccb8aff101a67f Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 14 Mar 2024 14:00:46 -0400 Subject: [PATCH 070/141] apf folder using new pcu except apfNumbering.cc --- apf/apfCavityOp.cc | 1 - apf/apfConstruct.cc | 59 +++++++++--------- apf/apfConvert.cc | 9 ++- apf/apfConvertTags.h | 33 +++++----- apf/apfFieldData.cc | 5 +- apf/apfH1Shapes.cc | 1 - apf/apfL2Shapes.cc | 1 - apf/apfMesh.cc | 1 - apf/apfMesh2.cc | 2 - apf/apfMesh2.h | 8 +-- apf/apfMigrate.cc | 115 ++++++++++++++++++----------------- apf/apfMixedNumbering.cc | 11 ++-- apf/apfMixedNumbering.h | 3 +- apf/apfNedelec.cc | 1 - apf/apfNumbering.cc | 21 +++---- apf/apfNumbering.h | 2 +- apf/apfPolyBasis1D.cc | 1 - apf/apfVerify.cc | 1 - apf/apfVtk.cc | 9 ++- apf/apfVtkPieceWiseFields.cc | 1 - phasta/phCook.cc | 2 +- pumi/pumi_mesh.cc | 4 +- pumi/pumi_numbering.cc | 2 +- test/mixedNumbering.cc | 14 ++--- test/repartition.cc | 14 +++-- 25 files changed, 156 insertions(+), 165 deletions(-) diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 9ad4d9637..51f2e01a4 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfCavityOp.h" #include "apf.h" #include "apfMesh2.h" diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index e461ecca1..218539c15 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -1,4 +1,3 @@ -#include #include "pcu_util.h" #include "apfConvert.h" #include "apfMesh2.h" @@ -43,12 +42,12 @@ static NewElements constructElements( return newElements; } -static Gid getMax(const GlobalToVert& globalToVert) +static Gid getMax(const GlobalToVert& globalToVert, Mesh2* m) { Gid max = -1; APF_CONST_ITERATE(GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return PCU_Max_Long(max); // this is type-dependent + return m->getPCU()->Max(max); // this is type-dependent } @@ -67,7 +66,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) } ifirst++; } - Gid max = getMax(globalToVert); // seems like we read this and know it already on every rank so why compute with global comm? + Gid max = getMax(globalToVert, m); // seems like we read this and know it already on every rank so why compute with global comm? ifirst=0; APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; @@ -111,7 +110,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) for each global id */ while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); int from = m->getPCU()->Sender(); Gid tmpL=gid - myOffset; // forcing 64 bit difference until we know it is safe int tmpI=tmpL; @@ -138,13 +137,13 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) model entity for that set of parts */ while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); int nparts; - PCU_COMM_UNPACK(nparts); + m->getPCU()->Unpack(nparts); Parts residence; for (int i = 0; i < nparts; ++i) { int part; - PCU_COMM_UNPACK(part); + m->getPCU()->Unpack(part); residence.insert(part); } MeshEntity* vert = globalToVert[gid]; @@ -175,9 +174,9 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert) m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); MeshEntity* remote; - PCU_COMM_UNPACK(remote); + m->getPCU()->Unpack(remote); int from = m->getPCU()->Sender(); MeshEntity* vert = globalToVert[gid]; m->addRemote(vert, from, remote); @@ -213,7 +212,7 @@ NewElements construct(Mesh2* m, const Gid* conn, int nelem, int etype, void setCoords(Mesh2* m, const double* coords, int nverts, GlobalToVert& globalToVert) { - Gid max = getMax(globalToVert); + Gid max = getMax(globalToVert, m); Gid total = max + 1; int peers = m->getPCU()->Peers(); int quotient = total / peers; @@ -251,8 +250,8 @@ void setCoords(Mesh2* m, const double* coords, int nverts, } m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(start); - PCU_COMM_UNPACK(n); // |||||| more in-place 64 bit math + m->getPCU()->Unpack(start); + m->getPCU()->Unpack(n); // |||||| more in-place 64 bit math m->getPCU()->Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double)); } @@ -270,7 +269,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); Gid from = m->getPCU()->Sender(); Gid tmpL=gid - myOffset; int tmpInt=tmpL; @@ -291,7 +290,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); double v[3]; m->getPCU()->Unpack(v, sizeof(v)); Vector3 vv(v); @@ -304,7 +303,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, void setMatches(Mesh2* m, const Gid* matches, int nverts, GlobalToVert& globalToVert) { - Gid max = getMax(globalToVert); + Gid max = getMax(globalToVert, m); Gid total = max + 1; int peers = m->getPCU()->Peers(); int quotient = total / peers; @@ -341,8 +340,8 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, } m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(start); - PCU_COMM_UNPACK(n); /// in-place 64 + m->getPCU()->Unpack(start); + m->getPCU()->Unpack(n); /// in-place 64 m->getPCU()->Unpack(&c[(start - myOffset)], n*sizeof(Gid)); } @@ -360,7 +359,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); int from = m->getPCU()->Sender(); tmpParts.at(gid - myOffset).push_back(from); } @@ -381,9 +380,9 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); Gid match; - PCU_COMM_UNPACK(match); + m->getPCU()->Unpack(match); PCU_ALWAYS_ASSERT(gid != match); PCU_ALWAYS_ASSERT(globalToVert.count(gid)); m->setLongTag(globalToVert[gid], matchGidTag, &match); @@ -409,9 +408,9 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, GidPtrs gidPtrs; while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); MeshEntity* vert; - PCU_COMM_UNPACK(vert); + m->getPCU()->Unpack(vert); int owner = m->getPCU()->Sender(); gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert)); } @@ -435,9 +434,9 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); // request from entity gid + m->getPCU()->Unpack(gid); // request from entity gid Gid matchGid; - PCU_COMM_UNPACK(matchGid); // requesting matched entity gid + m->getPCU()->Unpack(matchGid); // requesting matched entity gid MatchingPair mp(gid,matchGid); int from = m->getPCU()->Sender(); matchParts[mp].push_back(from); // store a list of the proceses that need the pair (entity gid, match gid) @@ -469,16 +468,16 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, m->getPCU()->Send(); while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); Gid matchGid; - PCU_COMM_UNPACK(matchGid); + m->getPCU()->Unpack(matchGid); size_t numMatches; - PCU_COMM_UNPACK(numMatches); + m->getPCU()->Unpack(numMatches); for(size_t i=0; igetPCU()->Unpack(owner); MeshEntity* match; - PCU_COMM_UNPACK(match); + m->getPCU()->Unpack(match); PCU_ALWAYS_ASSERT(globalToVert.count(gid)); MeshEntity* partner = globalToVert[gid]; PCU_ALWAYS_ASSERT(! (match == partner && owner == self) ); diff --git a/apf/apfConvert.cc b/apf/apfConvert.cc index 9b5227973..4c2d2c0ac 100644 --- a/apf/apfConvert.cc +++ b/apf/apfConvert.cc @@ -1,4 +1,3 @@ -#include #include "apf.h" #include "apfConvert.h" #include "apfMesh.h" @@ -187,9 +186,9 @@ class Converter while ( ! inMesh->getPCU()->Unpacked()) { MeshEntity* oldRight; - PCU_COMM_UNPACK(oldRight); + inMesh->getPCU()->Unpack(oldRight); MeshEntity* newLeft; - PCU_COMM_UNPACK(newLeft); + inMesh->getPCU()->Unpack(newLeft); MeshEntity *newRight = newFromOld[oldRight]; outMesh->addRemote(newRight, leftPart, newLeft); map_ent[newRight].push_back(leftPart); @@ -439,9 +438,9 @@ class Converter while ( ! inMesh->getPCU()->Unpacked()) { MeshEntity* oldRight; - PCU_COMM_UNPACK(oldRight); + inMesh->getPCU()->Unpack(oldRight); MeshEntity* newLeft; - PCU_COMM_UNPACK(newLeft); + inMesh->getPCU()->Unpack(newLeft); MeshEntity *newRight = newFromOld[oldRight]; outMesh->addMatch(newRight, leftPart, newLeft); } diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h index bdf916156..4905cdccf 100644 --- a/apf/apfConvertTags.h +++ b/apf/apfConvertTags.h @@ -1,17 +1,16 @@ #ifndef APF_CONVERT_TAGS_H #define APF_CONVERT_TAGS_H -#include #include "apf.h" #include "apfConvert.h" namespace { - static apf::Gid getMax(const apf::GlobalToVert& globalToVert) + static apf::Gid getMax(const apf::GlobalToVert& globalToVert, pcu::PCU *pcu_obj) { apf::Gid max = -1; APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return PCU_Max_Long(max); // this is type-dependent + return pcu_obj->Max(max); // this is type-dependent } template inline @@ -72,7 +71,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, const T* vals, const int entries, int nverts, GlobalToVert& globalToVert) { - apf::Gid max = getMax(globalToVert); + apf::Gid max = getMax(globalToVert, m->getPCU()); apf::Gid total = max + 1; int peers = m->getPCU()->Peers(); int quotient = total / peers; @@ -87,7 +86,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, This means we might need to send and recv some coords */ T* c = new T[mySize*entries]; - apf::Gid start = PCU_Exscan_Long(nverts); + apf::Gid start = m->getPCU()->Exscan(nverts); m->getPCU()->Begin(); apf::Gid tmpL=start / quotient; @@ -97,9 +96,9 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, tmpI=tmpL; int n = std::min(tmpI, nverts); while (nverts > 0) { - PCU_COMM_PACK(to, start); - PCU_COMM_PACK(to, n); - PCU_Comm_Pack(to, vals, n*entries*sizeof(T)); + m->getPCU()->Pack(to, start); + m->getPCU()->Pack(to, n); + m->getPCU()->Pack(to, vals, n*entries*sizeof(T)); nverts -= n; start += n; @@ -109,9 +108,9 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, } m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(start); - PCU_COMM_UNPACK(n); - PCU_Comm_Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(T)); + m->getPCU()->Unpack(start); + m->getPCU()->Unpack(n); + m->getPCU()->Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(T)); } /* Tell all the owners of the data what we need */ @@ -124,12 +123,12 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, tmpI=tmpL; int to = std::min(peers - 1, tmpI); // replaced int to = std::min(peers - 1, gid / quotient); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } m->getPCU()->Send(); while (m->getPCU()->Receive()) { apf::Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); int from = m->getPCU()->Sender(); tmpParts.at(gid - myOffset).push_back(from); } @@ -141,8 +140,8 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, for (size_t j = 0; j < parts.size(); ++j) { int to = parts[j]; apf::Gid gid = i + myOffset; - PCU_COMM_PACK(to, gid); - PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(T)); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, &c[i*entries], entries*sizeof(T)); } } m->getPCU()->Send(); @@ -150,8 +149,8 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, T* v = new T[entries]; while (m->getPCU()->Receive()) { apf::Gid gid; - PCU_COMM_UNPACK(gid); - PCU_Comm_Unpack(v, entries*sizeof(T)); + m->getPCU()->Unpack(gid); + m->getPCU()->Unpack(v, entries*sizeof(T)); setEntTag(m,t,globalToVert[gid],v); } delete [] v; diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc index a57629ead..db1f7dec0 100644 --- a/apf/apfFieldData.cc +++ b/apf/apfFieldData.cc @@ -1,4 +1,3 @@ -#include #include "apfFieldData.h" #include "apfShape.h" #include @@ -62,7 +61,7 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int n = f->countValuesOn(e); NewArray values(n); m->getPCU()->Unpack(&(values[0]),n*sizeof(T)); @@ -146,7 +145,7 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c { /* receive and add. we only care about correctness on the owners */ MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int n = f->countValuesOn(e); NewArray values(n); NewArray inValues(n); diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc index 673b2b781..53bc4b243 100644 --- a/apf/apfH1Shapes.cc +++ b/apf/apfH1Shapes.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc index 9f240b7d6..7c80095f3 100644 --- a/apf/apfL2Shapes.cc +++ b/apf/apfL2Shapes.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 490806561..fc20f2bdd 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfCoordData.h" diff --git a/apf/apfMesh2.cc b/apf/apfMesh2.cc index 090b00ad3..cd87fa9e0 100644 --- a/apf/apfMesh2.cc +++ b/apf/apfMesh2.cc @@ -1,9 +1,7 @@ -#include #include #include "apfMesh2.h" #include "apfField.h" #include "apf.h" - #include "apfShape.h" #include "apfTagData.h" #include "apfNumbering.h" diff --git a/apf/apfMesh2.h b/apf/apfMesh2.h index 7b5795ea2..2ade4062f 100644 --- a/apf/apfMesh2.h +++ b/apf/apfMesh2.h @@ -136,7 +136,7 @@ void migrateSilent(Mesh2* m, Migration* plan); This function globally sets the limit on migration, which causes any migration requests greater than the limit to be performed as several consecutive migrations. */ -void setMigrationLimit(size_t maxElements); +void setMigrationLimit(size_t maxElements, pcu::PCU *PCUObj); class Field; @@ -204,8 +204,8 @@ void unpackDataClone(Mesh2* m); // common functions for migration/ghosting/distribution typedef std::vector EntityVector; -void packParts(int to, Parts& parts); -void unpackParts(Parts& parts); +void packParts(int to, Parts& parts, pcu::PCU *PCUObj); +void unpackParts(Parts& parts, pcu::PCU *PCUObj); void moveEntities( Mesh2* m, EntityVector senders[4]); @@ -220,7 +220,7 @@ void reduceMatchingToSenders( Mesh2* m, EntityVector senders[4]); void getSenders(Mesh2* m,EntityVector affected[4],EntityVector senders[4]); -void split(Copies& remotes, Parts& parts, Parts& newParts); +void split(Copies& remotes, Parts& parts, Parts& newParts, pcu::PCU *PCUObj); // seol void packEntity(Mesh2* m, int to, MeshEntity* e, DynamicArray& tags, bool ghosting=false); diff --git a/apf/apfMigrate.cc b/apf/apfMigrate.cc index 55ad9d8c8..d17d6da96 100644 --- a/apf/apfMigrate.cc +++ b/apf/apfMigrate.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfMesh2.h" #include "apfCavityOp.h" #include "apf.h" @@ -73,7 +72,7 @@ static void getAffected( while (m->getPCU()->Receive()) { MeshEntity* entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); if ( ! m->hasTag(entity,tag)) { m->setIntTag(entity,tag,&dummy); @@ -142,9 +141,9 @@ void reduceMatchingToSenders( while ( ! m->getPCU()->Unpacked()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); MeshEntity* match; - PCU_COMM_UNPACK(match); + m->getPCU()->Unpack(match); /* all matches of the sender receive this. We only care that the senders themselves receive messages from only other senders matched to them, @@ -162,25 +161,25 @@ static Parts makeResidence(int part) return r; } -void packParts(int to, Parts& parts) +void packParts(int to, Parts& parts, pcu::PCU *PCUObj) { size_t n = parts.size(); - PCU_COMM_PACK(to,n); + PCUObj->Pack(to,n); APF_ITERATE(Parts,parts,it) { int p = *it; - PCU_COMM_PACK(to,p); + PCUObj->Pack(to,p); } } -void unpackParts(Parts& parts) +void unpackParts(Parts& parts, pcu::PCU *PCUObj) { size_t n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (size_t i=0;iUnpack(p); parts.insert(p); } } @@ -223,18 +222,18 @@ static void updateResidences( APF_ITERATE(Copies,remotes,rit) { m->getPCU()->Pack(rit->first,rit->second); - packParts(rit->first,newResidence); + packParts(rit->first,newResidence, m->getPCU()); } } m->getPCU()->Send(); while(m->getPCU()->Receive()) { MeshEntity* entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); Parts current; m->getResidence(entity,current); Parts incoming; - unpackParts(incoming); + unpackParts(incoming, m->getPCU()); unite(current,incoming); m->setResidence(entity,current); } @@ -250,10 +249,11 @@ static void updateResidences( void split( Copies& remotes, Parts& parts, - Parts& newParts) + Parts& newParts, + pcu::PCU *PCUObj) { APF_ITERATE(Parts,parts,it) - if (( ! remotes.count(*it))&&(*it != PCU_Comm_Self())) + if (( ! remotes.count(*it))&&(*it != PCUObj->Self())) newParts.insert(*it); } @@ -270,7 +270,7 @@ static void packCommon( m->getPCU()->Pack(to,modelTag); Parts residence; m->getResidence(e,residence); - packParts(to,residence); + packParts(to,residence, m->getPCU()); } void unpackCommon( @@ -279,12 +279,12 @@ void unpackCommon( ModelEntity*& c, Parts& residence) { - PCU_COMM_UNPACK(sender); + m->getPCU()->Unpack(sender); int modelType,modelTag; - PCU_COMM_UNPACK(modelType); - PCU_COMM_UNPACK(modelTag); + m->getPCU()->Unpack(modelType); + m->getPCU()->Unpack(modelTag); c = m->findModelEntity(modelType,modelTag); - unpackParts(residence); + unpackParts(residence, m->getPCU()); } static void packVertex( @@ -304,9 +304,9 @@ MeshEntity* unpackVertex( ModelEntity* c) { Vector3 point; - PCU_COMM_UNPACK(point); + m->getPCU()->Unpack(point); Vector3 param; - PCU_COMM_UNPACK(param); + m->getPCU()->Unpack(param); return m->createVertex(c,point,param); } @@ -345,12 +345,13 @@ static void packDownward(Mesh2* m, int to, MeshEntity* e) } static void unpackDownward( - Downward& entities) + Downward& entities, + pcu::PCU *PCUObj) { int n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (int i=0; i < n; ++i) - PCU_COMM_UNPACK(entities[i]); + PCUObj->Unpack(entities[i]); } static void packNonVertex( @@ -366,7 +367,7 @@ MeshEntity* unpackNonVertex( int type, ModelEntity* c) { Downward down; - unpackDownward(down); + unpackDownward(down, m->getPCU()); return m->createEntity(type,c,down); } @@ -428,13 +429,13 @@ void packRemotes( void unpackRemotes(Mesh2* m, MeshEntity* e) { size_t n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (size_t i=0; i < n; ++i) { int p; - PCU_COMM_UNPACK(p); + m->getPCU()->Unpack(p); MeshEntity* r; - PCU_COMM_UNPACK(r); + m->getPCU()->Unpack(r); m->addRemote(e, p, r); } } @@ -445,13 +446,13 @@ void unpackTags( DynamicArray& tags) { size_t n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); PCU_ALWAYS_ASSERT_VERBOSE(n<=tags.size(), "A tag was created that does not exist on all processes."); for (size_t t=0; t < n; ++t) { size_t i; - PCU_COMM_UNPACK(i); + m->getPCU()->Unpack(i); MeshTag* tag = tags[i]; int type = m->getTagType(tag); int size = m->getTagSize(tag); @@ -495,7 +496,7 @@ static MeshEntity* unpackEntity( { int from = m->getPCU()->Sender(); int type; - PCU_COMM_UNPACK(type); + m->getPCU()->Unpack(type); MeshEntity* sender; ModelEntity* c; Parts residence; @@ -527,7 +528,7 @@ static void sendEntities( Parts residence; m->getResidence(entity,residence); Parts sendTo; - split(remotes,residence,sendTo); + split(remotes,residence,sendTo,m->getPCU()); APF_ITERATE(Parts,sendTo,sit) packEntity(m,*sit,entity,tags); } @@ -570,9 +571,9 @@ static void receiveRemotes(Mesh2* m) while ( ! m->getPCU()->Unpacked()) { MeshEntity* sender; - PCU_COMM_UNPACK(sender); + m->getPCU()->Unpack(sender); MeshEntity* entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); PCU_ALWAYS_ASSERT(entity); m->addRemote(sender, from, entity); } @@ -603,28 +604,30 @@ static void getNewCopies( static void packCopies( int to, - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n = copies.size(); - PCU_COMM_PACK(to,n); + PCUObj->Pack(to,n); APF_ITERATE(Copies,copies,it) { - PCU_COMM_PACK(to,it->first); - PCU_COMM_PACK(to,it->second); + PCUObj->Pack(to,it->first); + PCUObj->Pack(to,it->second); } } static void unpackCopies( - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (int i=0; i < n; ++i) { int part; - PCU_COMM_UNPACK(part); + PCUObj->Unpack(part); MeshEntity* remote; - PCU_COMM_UNPACK(remote); + PCUObj->Unpack(remote); PCU_ALWAYS_ASSERT(remote); copies[part]=remote; } @@ -646,7 +649,7 @@ static void bcastRemotes( APF_ITERATE(Copies,allRemotes,rit) { m->getPCU()->Pack(rit->first,rit->second); - packCopies(rit->first,newCopies); + packCopies(rit->first,newCopies, m->getPCU()); } newCopies.erase(rank); m->setRemotes(e,newCopies); @@ -655,9 +658,9 @@ static void bcastRemotes( while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Copies copies; - unpackCopies(copies); + unpackCopies(copies, m->getPCU()); copies.erase(rank); m->setRemotes(e,copies); } @@ -721,13 +724,13 @@ static void updateSenderMatching( /* pack the remote copies to itself, then add itself to the copies if it remains */ m->getPCU()->Pack(self,e); - packCopies(self,copies); + packCopies(self,copies, m->getPCU()); if (residence.count(self)) copies[self]=e; for (size_t j=0; j < matches.getSize(); ++j) { /* pack all copies to matched senders */ m->getPCU()->Pack(matches[j].peer,matches[j].entity); - packCopies(matches[j].peer,copies); + packCopies(matches[j].peer,copies, m->getPCU()); } } /* destroy all matching in the entire affected set. @@ -740,9 +743,9 @@ static void updateSenderMatching( while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Copies copies; - unpackCopies(copies); + unpackCopies(copies, m->getPCU()); APF_ITERATE(Copies,copies,cit) m->addMatch(e,cit->first,cit->second); } @@ -792,14 +795,14 @@ static void bcastMatching( while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Match match; size_t n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (size_t i=0; i < n; ++i) { - PCU_COMM_UNPACK(match.peer); - PCU_COMM_UNPACK(match.entity); + m->getPCU()->Unpack(match.peer); + m->getPCU()->Unpack(match.entity); if ( ! ((match.peer == self)&& (match.entity == e))) m->addMatch(e,match.peer,match.entity); @@ -858,10 +861,10 @@ static void migrate1(Mesh2* m, Migration* plan) const size_t maxMigrationLimit = 10*1000*1000; static size_t migrationLimit = maxMigrationLimit; -void setMigrationLimit(size_t maxElements) +void setMigrationLimit(size_t maxElements, pcu::PCU *PCUObj) { if( maxElements >= maxMigrationLimit ) { - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) lion_eprint(1, "ERROR requested migration limit exceeds" " %lu... exiting\n", maxMigrationLimit); abort(); diff --git a/apf/apfMixedNumbering.cc b/apf/apfMixedNumbering.cc index 5cc1f354a..363c1e509 100644 --- a/apf/apfMixedNumbering.cc +++ b/apf/apfMixedNumbering.cc @@ -8,7 +8,6 @@ #include "apfNumbering.h" #include "apfNumberingClass.h" #include "apfShape.h" -#include #include #include #include @@ -228,8 +227,9 @@ static int number_ghost( static void globalize( int dofs, std::vector const& owned, - std::vector& global) { - long start = PCU_Exscan_Long(long(dofs)); + std::vector& global, + pcu::PCU *PCUObj) { + long start = PCUObj->Exscan(long(dofs)); DynamicArray nodes; for (size_t f=0; f < global.size(); ++f) { getNodes(owned[f], nodes); @@ -315,10 +315,11 @@ int numberGhost( void makeGlobal( std::vector& owned, - std::vector& global) { + std::vector& global, + pcu::PCU *PCUObj) { int dofs = countDOFs(owned); create_global(owned, global); - globalize(dofs, owned, global); + globalize(dofs, owned, global, PCUObj); } } diff --git a/apf/apfMixedNumbering.h b/apf/apfMixedNumbering.h index a8429aa09..87494a5c4 100644 --- a/apf/apfMixedNumbering.h +++ b/apf/apfMixedNumbering.h @@ -69,7 +69,8 @@ int numberGhost( * \param global The output global numberings. */ void makeGlobal( std::vector& owned, - std::vector& global); + std::vector& global, + pcu::PCU *PCUObj); } diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc index deb9b7d3b..5ef3b5065 100644 --- a/apf/apfNedelec.cc +++ b/apf/apfNedelec.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfNumbering.cc b/apf/apfNumbering.cc index 1ecb11af1..6e0d78e30 100644 --- a/apf/apfNumbering.cc +++ b/apf/apfNumbering.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfNumbering.h" #include "apfNumberingClass.h" #include "apfField.h" @@ -414,7 +413,7 @@ static void synchronizeEntitySet( while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); set.insert(e); } } @@ -532,9 +531,9 @@ int getElementNumbers(GlobalNumbering* n, MeshEntity* e, return n->getData()->getElementData(e,numbers); } -static long exscan(long x) +static long exscan(long x, pcu::PCU *PCUObj) { - return PCU_Exscan_Long(x); + return PCUObj->Exscan(x); } template @@ -557,26 +556,26 @@ class Globalizer : public FieldOp } return false; } - void run(NumberingOf* n) + void run(NumberingOf* n, pcu::PCU *PCUObj) { numbering = n; data = n->getData(); start = countFieldNodes(n); - start = exscan(start); + start = exscan(start, PCUObj); apply(n); } }; -static void globalize(GlobalNumbering* n) +static void globalize(GlobalNumbering* n, pcu::PCU *PCUObj) { Globalizer g; - g.run(n); + g.run(n, PCUObj); } -void globalize(Numbering* n) +void globalize(Numbering* n, pcu::PCU *PCUObj) { Globalizer g; - g.run(n); + g.run(n, PCUObj); } GlobalNumbering* makeGlobal(Numbering* n, bool destroy) @@ -606,7 +605,7 @@ GlobalNumbering* makeGlobal(Numbering* n, bool destroy) } if (destroy) apf::destroyNumbering(n); - globalize(gn); + globalize(gn, m->getPCU()); return gn; } diff --git a/apf/apfNumbering.h b/apf/apfNumbering.h index 36cf3f46d..bdfbcd91d 100644 --- a/apf/apfNumbering.h +++ b/apf/apfNumbering.h @@ -222,7 +222,7 @@ void getNodes(GlobalNumbering* n, DynamicArray& nodes); number the vertices and elements of a mesh */ MeshTag* reorder(Mesh* mesh, const char* name); -void globalize(Numbering* n); +void globalize(Numbering* n, pcu::PCU *PCUObj); /** \brief number all components by simple iteration */ int naiveOrder(Numbering * num, Sharing * sharing = NULL); diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc index 9af87c6e9..1a86c6a79 100644 --- a/apf/apfPolyBasis1D.cc +++ b/apf/apfPolyBasis1D.cc @@ -11,7 +11,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc index 0af98cd81..6b2969157 100644 --- a/apf/apfVerify.cc +++ b/apf/apfVerify.cc @@ -1,4 +1,3 @@ -#include #include "apfMesh.h" #include "apf.h" #include diff --git a/apf/apfVtk.cc b/apf/apfVtk.cc index 8915dd55d..09e30ddb8 100644 --- a/apf/apfVtk.cc +++ b/apf/apfVtk.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfMesh.h" @@ -46,7 +45,7 @@ static bool shouldPrint( bool isPrintable(FieldBase* f) { HasAll op; - return PCU_And(op.run(f)); + return f->getMesh()->getPCU()->And(op.run(f)); } static bool isNodal(FieldBase* f) @@ -254,9 +253,9 @@ static std::string getRelativePathPSource(int id) return ss.str(); } -static void writePSources(std::ostream& file) +static void writePSources(std::ostream& file, pcu::PCU *PCUObj) { - for (int i=0; i < PCU_Comm_Peers(); ++i) + for (int i=0; i < PCUObj->Peers(); ++i) { std::string fileName = stripPath(getPieceFileName(i)); std::string fileNameAndPath = getRelativePathPSource(i) + fileName; @@ -282,7 +281,7 @@ static void writePvtuFile(const char* prefix, writePPoints(file,m->getCoordinateField(),isWritingBinary); writePPointData(file,m,writeFields,isWritingBinary); writePCellData(file, m, writeFields, isWritingBinary, cellDim); - writePSources(file); + writePSources(file, m->getPCU()); file << "\n"; file << "\n"; } diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc index f57527b0a..d019cee2a 100644 --- a/apf/apfVtkPieceWiseFields.cc +++ b/apf/apfVtkPieceWiseFields.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfMesh.h" diff --git a/phasta/phCook.cc b/phasta/phCook.cc index fe9636a8d..325cacafd 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -99,7 +99,7 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, ph::attachZeroSolution(in, m); if (in.buildMapping) ph::buildMapping(m); - apf::setMigrationLimit(SIZET(in.elementsPerMigration)); + apf::setMigrationLimit(SIZET(in.elementsPerMigration), m->getPCU()); if (in.adaptFlag) ph::adapt(in, m); if (in.tetrahedronize) diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 728eb619e..f04633638 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -959,7 +959,7 @@ static void distr_updateResidences(pMesh m, APF_ITERATE(Copies,remotes,rit) { m->getPCU()->Pack(rit->first,rit->second); - apf::packParts(rit->first,newResidence); + apf::packParts(rit->first,newResidence, m->getPCU()); } } m->getPCU()->Send(); @@ -970,7 +970,7 @@ static void distr_updateResidences(pMesh m, Parts current; m->getResidence(entity,current); Parts incoming; - apf::unpackParts(incoming); + apf::unpackParts(incoming, m->getPCU()); apf::unite(current,incoming); m->setResidence(entity,current); } diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index e0b07c447..6ed34588f 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -61,7 +61,7 @@ pNumbering pumi_numbering_createGlobal(pMesh m, const char* name, pShape s, pOwn if (!s) s= m->getShape(); n = numberOwnedNodes(m, name, s, o); - apf::globalize(n); + apf::globalize(n, m->getPCU()); apf::synchronizeFieldData(n->getData(), o, false); //synchronize(n, o); return n; } diff --git a/test/mixedNumbering.cc b/test/mixedNumbering.cc index 7c3fe5533..3613eb5e8 100644 --- a/test/mixedNumbering.cc +++ b/test/mixedNumbering.cc @@ -9,6 +9,7 @@ #include #include #include +#include static void test_numbering(apf::Mesh* m) { apf::FieldShape* S2 = apf::getSerendipity(); @@ -25,14 +26,14 @@ static void test_numbering(apf::Mesh* m) { std::vector global; int num_owned = apf::numberOwned(fields, owned); int num_ghost = apf::numberGhost(fields, ghost); - apf::makeGlobal(owned, global); + apf::makeGlobal(owned, global, m->getPCU()); for (size_t n=0; n < owned.size(); ++n) apf::destroyNumbering(owned[n]); for (size_t n=0; n < global.size(); ++n) apf::synchronize(global[n]); - PCU_Debug_Open(); - PCU_Debug_Print("number owned: %d\n", num_owned); - PCU_Debug_Print("number ghost: %d\n", num_ghost); + m->getPCU()->DebugOpen(); + m->getPCU()->DebugPrint("number owned: %d\n", num_owned); + m->getPCU()->DebugPrint("number ghost: %d\n", num_ghost); } static void write_output(apf::Mesh* m, const char* out) { @@ -49,15 +50,14 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); apf::reorderMdsMesh(m); test_numbering(m); write_output(m, argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); MPI_Finalize(); } diff --git a/test/repartition.cc b/test/repartition.cc index ca3913d64..e43bbc6f3 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -9,6 +9,7 @@ #include #include #include +#include namespace { @@ -18,8 +19,8 @@ const char* outFile = 0; int inputPartCount = 1; struct CreateGroupCommResult{ - bool isOriginal; - pcu::PCU *group_pcu_obj; + bool isOriginal; + pcu::PCU *group_pcu_obj; }; void freeMesh(apf::Mesh* m) @@ -97,18 +98,19 @@ void balance(apf::Mesh2* m) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - pcu::PCU *expanded_pcu_obj = new pcu::PCU(MPI_COMM_WORLD); + //pcu::PCU *expanded_pcu_obj = new pcu::PCU(MPI_COMM_WORLD); + auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv,expanded_pcu_obj); + getConfig(argc,argv,expanded_pcu_obj.get()); gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = 0; - CreateGroupCommResult result = createGroupComm(expanded_pcu_obj); + CreateGroupCommResult result = createGroupComm(expanded_pcu_obj.get()); if (result.isOriginal) m = apf::loadMdsMesh(g, meshFile, result.group_pcu_obj); - m = apf::expandMdsMesh(m, g, inputPartCount, expanded_pcu_obj); + m = apf::expandMdsMesh(m, g, inputPartCount, expanded_pcu_obj.get()); balance(m); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); From bdce405b63b235132bc28d12e11bf14e3607392c Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 15 Mar 2024 11:48:08 -0400 Subject: [PATCH 071/141] ma folder changed to use new pcu, changed some uses of print() in crv folder --- crv/crvAdapt.cc | 8 +++---- crv/crvShape.cc | 10 ++++----- ma/ma.cc | 18 ++++++++-------- ma/maAdapt.cc | 5 ++--- ma/maAdapt.h | 2 +- ma/maBalance.cc | 2 +- ma/maCoarsen.cc | 2 +- ma/maInput.cc | 47 +++++++++++++++++++++--------------------- ma/maLayer.cc | 6 +++--- ma/maLayerCoarsen.cc | 2 +- ma/maLayerSnap.cc | 12 +++++------ ma/maMatch.cc | 15 +++++++------- ma/maRefine.cc | 2 +- ma/maShape.cc | 14 ++++++------- ma/maSnap.cc | 2 +- ma/maTetrahedronize.cc | 6 +++--- test/repartition.cc | 1 - 17 files changed, 75 insertions(+), 79 deletions(-) diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc index 2b0121b47..43b85a184 100644 --- a/crv/crvAdapt.cc +++ b/crv/crvAdapt.cc @@ -80,7 +80,7 @@ static void refine(ma::Adapt* a) } splitEdges(a); double t1 = pcu::Time(); - ma::print("split %li edges in %f seconds",count,t1-t0); + ma::print("split %li edges in %f seconds", a->mesh->getPCU(), count, t1-t0); } int getValidityTag(ma::Mesh* m, ma::Entity* e, @@ -198,7 +198,7 @@ void adapt(ma::Input* in) fail("mesh must be bezier to adapt\n"); in->shapeHandler = crv::getShapeHandler; - ma::print("Curved Adaptation Version 2.0 !"); + ma::print("Curved Adaptation Version 2.0 !", in->mesh->getPCU()); double t0 = pcu::Time(); ma::validateInput(in); Adapt* a = new Adapt(in); @@ -208,7 +208,7 @@ void adapt(ma::Input* in) for (int i=0; i < in->maximumIterations; ++i) { - ma::print("iteration %d",i); + ma::print("iteration %d", a->mesh->getPCU(), i); ma::coarsen(a); ma::midBalance(a); crv::refine(a); @@ -228,7 +228,7 @@ void adapt(ma::Input* in) ma::printQuality(a); ma::postBalance(a); double t1 = pcu::Time(); - ma::print("mesh adapted in %f seconds",t1-t0); + ma::print("mesh adapted in %f seconds", a->mesh->getPCU(), t1-t0); apf::printStats(a->mesh); crv::clearTags(a); delete a; diff --git a/crv/crvShape.cc b/crv/crvShape.cc index 73f649e29..08ee452a2 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -715,7 +715,7 @@ int fixLargeBoundaryAngles(Adapt* a) splitEdges(a); double t1 = pcu::Time(); ma::print("split %d boundary edges with " - "large angles in %f seconds",count,t1-t0); + "large angles in %f seconds", a->mesh->getPCU(), count, t1-t0); return 0; } @@ -735,7 +735,7 @@ static void collapseInvalidEdges(Adapt* a) successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); ma::print("Collapsed %d bad edges " - "in %f seconds",successCount, t1-t0); + "in %f seconds", m->getPCU(), successCount, t1-t0); } static void swapInvalidEdges(Adapt* a) @@ -745,7 +745,7 @@ static void swapInvalidEdges(Adapt* a) ma::applyOperator(a,&es); double t1 = pcu::Time(); ma::print("Swapped %d bad edges " - "in %f seconds",es.ns, t1-t0); + "in %f seconds", a->mesh->getPCU(), es.ns, t1-t0); } static void repositionInvalidEdges(Adapt* a) @@ -755,7 +755,7 @@ static void repositionInvalidEdges(Adapt* a) ma::applyOperator(a,&es); double t1 = pcu::Time(); ma::print("Repositioned %d bad edges " - "in %f seconds",es.nr, t1-t0); + "in %f seconds", a->mesh->getPCU(), es.nr, t1-t0); } int fixInvalidEdges(Adapt* a) @@ -850,7 +850,7 @@ void fixCrvElementShapes(Adapt* a) ++i; } while(count < prev_count && i < 6); // the second conditions is to make sure this does not take long double t1 = pcu::Time(); - ma::print("bad shapes down from %d to %d in %f seconds", + ma::print("bad shapes down from %d to %d in %f seconds", a->mesh->getPCU(), originalCount,count,t1-t0); a->input->shouldForceAdaptation = false; } diff --git a/ma/ma.cc b/ma/ma.cc index 23bb1ff41..9ff1e14ed 100644 --- a/ma/ma.cc +++ b/ma/ma.cc @@ -22,14 +22,14 @@ namespace ma { void adapt(Input* in) { - print("version 2.0 !"); double t0 = pcu::Time(); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); + print("version 2.0 !", a->mesh->getPCU()); for (int i = 0; i < in->maximumIterations; ++i) { - print("iteration %d",i); + print("iteration %d", a->mesh->getPCU(), i); coarsen(a); coarsenLayer(a); midBalance(a); @@ -51,7 +51,7 @@ void adapt(Input* in) delete in->solutionTransfer; delete in; double t1 = pcu::Time(); - print("mesh adapted in %f seconds",t1-t0); + print("mesh adapted in %f seconds", m->getPCU(), t1-t0); apf::printStats(m); } @@ -62,14 +62,14 @@ void adapt(const Input* in) void adaptVerbose(Input* in, bool verbose) { - print("version 2.0 - dev !"); double t0 = pcu::Time(); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); + print("version 2.0 - dev !", a->mesh->getPCU()); for (int i = 0; i < in->maximumIterations; ++i) { - print("iteration %d",i); + print("iteration %d",a->mesh->getPCU(), i); coarsen(a); if (verbose && in->shouldCoarsen) ma_dbg::dumpMeshWithQualities(a,i,"after_coarsen"); @@ -99,14 +99,14 @@ void adaptVerbose(Input* in, bool verbose) */ int count = 0; double lMax = ma::getMaximumEdgeLength(a->mesh, a->sizeField); - print("Maximum (metric) edge length in the mesh is %f", lMax); + print("Maximum (metric) edge length in the mesh is %f", a->mesh->getPCU(), lMax); while (lMax > 1.5) { - print("%dth additional refine-snap call", count); + print("%dth additional refine-snap call", a->mesh->getPCU(), count); refine(a); snap(a); lMax = ma::getMaximumEdgeLength(a->mesh, a->sizeField); count++; - print("Maximum (metric) edge length in the mesh is %f", lMax); + print("Maximum (metric) edge length in the mesh is %f", a->mesh->getPCU(), lMax); if (count > 5) break; } if (verbose) @@ -125,7 +125,7 @@ void adaptVerbose(Input* in, bool verbose) delete in->solutionTransfer; delete in; double t1 = pcu::Time(); - print("mesh adapted in %f seconds",t1-t0); + print("mesh adapted in %f seconds", m->getPCU(), t1-t0); apf::printStats(m); } diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index 6beb7bad8..2a1feb2e7 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include #include "maAdapt.h" #include "maInput.h" @@ -471,9 +470,9 @@ void clearBuildCallback(Adapt* a) a->buildCallback = 0; } -void print(const char* format, ...) +void print(const char* format, pcu::PCU *PCUObj, ...) { - if (PCU_Comm_Self()) + if (PCUObj->Self()) return; lion_oprint(1,"\nMeshAdapt: "); va_list ap; diff --git a/ma/maAdapt.h b/ma/maAdapt.h index 81a943e0a..06ea608c7 100644 --- a/ma/maAdapt.h +++ b/ma/maAdapt.h @@ -164,7 +164,7 @@ Entity* rebuildElement( void setBuildCallback(Adapt* a, apf::BuildCallback* cb); void clearBuildCallback(Adapt* a); -void print(const char* format, ...) __attribute__((format(printf,1,2))); +void print(const char* format, pcu::PCU *PCUObj, ...) __attribute__((format(printf,1,3))); void setFlagOnClosure(Adapt* a, Entity* e, int flag); void syncFlag(Adapt* a, int dimension, int flag); diff --git a/ma/maBalance.cc b/ma/maBalance.cc index f446e44a6..5f4b10a5f 100644 --- a/ma/maBalance.cc +++ b/ma/maBalance.cc @@ -119,7 +119,7 @@ void printEntityImbalance(Mesh* m) double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); double p = (imbalance[m->getDimension()]-1)*100; - print("element imbalance %.0f%% of average",p); + print("element imbalance %.0f%% of average", m->getPCU(), p); } double estimateWeightedImbalance(Adapt* a) diff --git a/ma/maCoarsen.cc b/ma/maCoarsen.cc index a8c2ece3e..4f7607901 100644 --- a/ma/maCoarsen.cc +++ b/ma/maCoarsen.cc @@ -249,7 +249,7 @@ bool coarsen(Adapt* a) } successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); - print("coarsened %li edges in %f seconds",successCount,t1-t0); + print("coarsened %li edges in %f seconds", m->getPCU(), successCount,t1-t0); return true; } diff --git a/ma/maInput.cc b/ma/maInput.cc index 9b1c9d25f..1c159ef4b 100644 --- a/ma/maInput.cc +++ b/ma/maInput.cc @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -66,9 +65,9 @@ void setDefaultValues(Input* in) in->debugFolder = nullptr; } -void rejectInput(const char* str) +void rejectInput(const char* str, pcu::PCU *PCUObj) { - if (PCU_Comm_Self() != 0) + if (PCUObj->Self() != 0) return; lion_eprint(1,"MeshAdapt input error:\n"); lion_eprint(1,"%s\n",str); @@ -90,62 +89,62 @@ static bool moreThanOneOptionIsTrue(bool op1, bool op2, bool op3) void validateInput(Input* in) { if ( ! in->sizeField) - rejectInput("no size field"); + rejectInput("no size field", in->mesh->getPCU()); if ( ! in->solutionTransfer) - rejectInput("no solution transfer object"); + rejectInput("no solution transfer object", in->mesh->getPCU()); if (in->maximumIterations < 0) - rejectInput("negative maximum iteration count"); + rejectInput("negative maximum iteration count", in->mesh->getPCU()); if (in->maximumIterations > 10) - rejectInput("unusually high maximum iteration count"); + rejectInput("unusually high maximum iteration count", in->mesh->getPCU()); if (in->shouldSnap &&( ! in->mesh->canSnap())) rejectInput("user requested snapping " - "but the geometric model does not support it"); + "but the geometric model does not support it", in->mesh->getPCU()); if (in->shouldTransferParametric &&( ! in->mesh->canSnap())) rejectInput("user requested parametric coordinate transfer " - "but the geometric model does not support it"); + "but the geometric model does not support it", in->mesh->getPCU()); if (in->shouldTransferToClosestPoint &&( ! in->mesh->canSnap())) rejectInput("user requested transfer to closest point on model" - "but the geometric model does not support it"); + "but the geometric model does not support it", in->mesh->getPCU()); if (in->shouldSnap && ( ! (in->shouldTransferParametric || in->shouldTransferToClosestPoint))) - rejectInput("snapping requires parametric coordinate transfer or transfer to closest point"); + rejectInput("snapping requires parametric coordinate transfer or transfer to closest point", in->mesh->getPCU()); if ((in->mesh->hasMatching()) &&( ! in->shouldHandleMatching)) - rejectInput("the mesh has matching entities but matched support is off"); + rejectInput("the mesh has matching entities but matched support is off", in->mesh->getPCU()); if (in->shouldHandleMatching && in->shouldFixShape) rejectInput("user requested matched mesh handling and shape correction " - "but shape correction does not support matching yet"); + "but shape correction does not support matching yet", in->mesh->getPCU()); if (in->goodQuality < 0.0) - rejectInput("negative desired element quality"); + rejectInput("negative desired element quality", in->mesh->getPCU()); if (in->goodQuality > 1.0) - rejectInput("desired element quality greater than one"); + rejectInput("desired element quality greater than one", in->mesh->getPCU()); if (in->validQuality < 0.0) - rejectInput("negative minimum element quality"); + rejectInput("negative minimum element quality", in->mesh->getPCU()); if (in->maximumImbalance < 1.0) - rejectInput("maximum imbalance less than 1.0"); + rejectInput("maximum imbalance less than 1.0", in->mesh->getPCU()); if (in->maximumEdgeRatio < 1.0) - rejectInput("maximum tet edge ratio less than one"); + rejectInput("maximum tet edge ratio less than one", in->mesh->getPCU()); if (moreThanOneOptionIsTrue( in->shouldRunPreZoltan, in->shouldRunPreZoltanRib, in->shouldRunPreParma)) - rejectInput("only one of Zoltan, ZoltanRib, and Parma PreBalance options can be set to true!"); + rejectInput("only one of Zoltan, ZoltanRib, and Parma PreBalance options can be set to true!", in->mesh->getPCU()); if (moreThanOneOptionIsTrue( in->shouldRunPostZoltan, in->shouldRunPostZoltanRib, in->shouldRunPostParma)) - rejectInput("only one of Zoltan, ZoltanRib, and Parma PostBalance options can be set to true!"); + rejectInput("only one of Zoltan, ZoltanRib, and Parma PostBalance options can be set to true!", in->mesh->getPCU()); if (in->shouldRunMidZoltan && in->shouldRunMidParma) - rejectInput("only one of Zoltan and Parma MidBalance options can be set to true!"); + rejectInput("only one of Zoltan and Parma MidBalance options can be set to true!", in->mesh->getPCU()); #ifndef PUMI_HAS_ZOLTAN if (in->shouldRunPreZoltan || in->shouldRunPreZoltanRib || in->shouldRunMidZoltan) - rejectInput("core is not compiled with Zoltan. Use a different balancer or compile core with ENABLE_ZOLTAN=ON!"); + rejectInput("core is not compiled with Zoltan. Use a different balancer or compile core with ENABLE_ZOLTAN=ON!", in->mesh->getPCU()); #endif } @@ -157,12 +156,12 @@ static void updateMaxIterBasedOnSize(Mesh* m, Input* in) if (iter >= 10) { print("ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations,\n" " which is equal to or larger than the maximum of 10 allowed.\n" - " Setting the number of iteration to 10!", iter); + " Setting the number of iteration to 10!", m->getPCU(), iter); in->maximumIterations = 10; } else { print("ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations.\n" - " Setting the number of iteration to %d!", iter, iter+1); + " Setting the number of iteration to %d!", m->getPCU(), iter, iter+1); in->maximumIterations = iter+1; } } diff --git a/ma/maLayer.cc b/ma/maLayer.cc index 23636699c..12814d44c 100644 --- a/ma/maLayer.cc +++ b/ma/maLayer.cc @@ -99,7 +99,7 @@ void resetLayer(Adapt* a) return; freezeLayer(a); double t1 = pcu::Time(); - print("marked %ld layer elements in %f seconds", n, t1 - t0); + print("marked %ld layer elements in %f seconds", a->mesh->getPCU(), n, t1 - t0); } void findLayerBase(Adapt* a) @@ -148,7 +148,7 @@ void allowSplitInLayer(Adapt* a) if (getFlag(a,e,LAYER)) clearFlag(a,e,DONT_SPLIT); m->end(it); - print("allowing layer refinement"); + print("allowing layer refinement", m->getPCU()); } void collectForLayerRefine(Refine* r) @@ -188,7 +188,7 @@ void checkLayerShape(Mesh* m, const char* key) m->end(it); n = m->getPCU()->Add(n); double t1 = pcu::Time(); - print("%s: checked layer quality in %f seconds: %ld unsafe elements", key, t1 - t0, n); + print("%s: checked layer quality in %f seconds: %ld unsafe elements", m->getPCU(), key, t1 - t0, n); } } diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index 4abd2e498..2c16491f0 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -283,7 +283,7 @@ bool coarsenLayer(Adapt* a) successCount += collapseAllStacks(a, d); } double t1 = pcu::Time(); - print("coarsened %li layer edges in %f seconds",successCount,t1-t0); + print("coarsened %li layer edges in %f seconds", m->getPCU(), successCount,t1-t0); resetLayer(a); return true; } diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index c3a3dbd9a..ca8611908 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -274,7 +274,7 @@ static long snapAllCurves(Adapt* a, Tag* snapTag) LayerSnapper op(a, snapTag); crawlLayers(&op); double t1 = pcu::Time(); - print("snapped %ld curves in %f seconds", op.ncurves, t1 - t0); + print("snapped %ld curves in %f seconds", a->mesh->getPCU(), op.ncurves, t1 - t0); return op.ncurves; } @@ -402,9 +402,9 @@ static bool checkForUnsnap(Adapt* a, Tag* snapTag) bool notOk = a->mesh->getPCU()->Or(op.foundAnything); double t1 = pcu::Time(); if (notOk) - print("checked snapped curves in %f seconds, found some to unsnap", t1 - t0); + print("checked snapped curves in %f seconds, found some to unsnap", a->mesh->getPCU(), t1 - t0); else - print("checked snapped curves in %f seconds, all ok", t1 - t0); + print("checked snapped curves in %f seconds, all ok", a->mesh->getPCU(), t1 - t0); return notOk; } @@ -437,7 +437,7 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) PCU_ALWAYS_ASSERT(m->hasTag(v, snapTag)); } n = m->getPCU()->Add(n); - print("fed back unsnap flag from %ld tops", n); + print("fed back unsnap flag from %ld tops", m->getPCU(), n); } /* for each layer curve whose base vertex @@ -528,7 +528,7 @@ static long unsnapMarkedCurves(Adapt* a, Tag* snapTag) Unsnapper op(a, snapTag); crawlLayers(&op); double t1 = pcu::Time(); - print("unsnapped %ld curves in %f seconds", op.ncurves, t1 - t0); + print("unsnapped %ld curves in %f seconds", a->mesh->getPCU(), op.ncurves, t1 - t0); return op.ncurves; } @@ -550,7 +550,7 @@ void snapLayer(Adapt* a, Tag* snapTag) } delete l; double t1 = pcu::Time(); - print("finished snapping %ld of %ld layer curves in %f seconds", + print("finished snapping %ld of %ld layer curves in %f seconds", a->mesh->getPCU(), nsnapped - nunsnapped, nsnapped, t1 - t0); } diff --git a/ma/maMatch.cc b/ma/maMatch.cc index e62758cf4..020daca43 100644 --- a/ma/maMatch.cc +++ b/ma/maMatch.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maMatch.h" #include "maMesh.h" #include "maAdapt.h" @@ -15,16 +14,16 @@ namespace ma { -static void packSplits(int to, EntityArray& splits) +static void packSplits(int to, EntityArray& splits, pcu::PCU *PCUObj) { - PCU_Comm_Pack(to, + PCUObj->Pack(to, static_cast(&(splits[0])), splits.getSize()*sizeof(Entity*)); } -static void unpackSplits(EntityArray& splits) +static void unpackSplits(EntityArray& splits, pcu::PCU *PCUObj) { - PCU_Comm_Unpack( + PCUObj->Unpack( static_cast(&(splits[0])), splits.getSize()*sizeof(Entity*)); } @@ -50,7 +49,7 @@ void matchNewElements(Refine* r) int to = matches[i].peer; Entity* match = matches[i].entity; m->getPCU()->Pack(to,match); - packSplits(to,splits); + packSplits(to,splits,m->getPCU()); } } m->getPCU()->Send(); @@ -65,7 +64,7 @@ void matchNewElements(Refine* r) m->getIntTag(e,r->numberTag,&number); EntityArray& splits = r->newEntities[d][number]; EntityArray remoteSplits(splits.getSize()); - unpackSplits(remoteSplits); + unpackSplits(remoteSplits,m->getPCU()); for (size_t i=0; i < splits.getSize(); ++i) m->addMatch(splits[i],from,remoteSplits[i]); if (d==2) ++face_count; @@ -73,7 +72,7 @@ void matchNewElements(Refine* r) } } face_count = m->getPCU()->Add(face_count); - print("updated matching for %li faces",face_count); + print("updated matching for %li faces", m->getPCU(), face_count); } void preventMatchedCavityMods(Adapt* a) diff --git a/ma/maRefine.cc b/ma/maRefine.cc index 1d07bbaf0..661377d59 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -438,7 +438,7 @@ bool refine(Adapt* a) destroySplitElements(r); forgetNewEntities(r); double t1 = pcu::Time(); - print("refined %li edges in %f seconds",count,t1-t0); + print("refined %li edges in %f seconds",a->mesh->getPCU(),count,t1-t0); resetLayer(a); if (a->hasLayer) checkLayerShape(a->mesh, "after refinement"); diff --git a/ma/maShape.cc b/ma/maShape.cc index aae547aee..845bf21ac 100644 --- a/ma/maShape.cc +++ b/ma/maShape.cc @@ -820,7 +820,7 @@ void fixElementShapes(Adapt* a) if ( ! count) break; prev_count = count; - print("--iter %d of shape correction loop: #bad elements %d", iter, count); + print("--iter %d of shape correction loop: #bad elements %d", a->mesh->getPCU(), iter, count); time = fixLargeAngles(a); /* We need to snap the new verts as soon as they are * created (to avoid future problems). At the moment @@ -830,20 +830,20 @@ void fixElementShapes(Adapt* a) if (a->mesh->getDimension() == 3) snap(a); count = markBadQuality(a); - print("--fixLargeAngles in %f seconds: #bad elements %d",time,count); + print("--fixLargeAngles in %f seconds: #bad elements %d", a->mesh->getPCU(), time,count); time = fixShortEdgeElements(a); count = markBadQuality(a); - print("--fixShortEdgeElements in %f seconds: #bad elements %d",time,count); + print("--fixShortEdgeElements in %f seconds: #bad elements %d", a->mesh->getPCU(), time,count); if (count >= prev_count) unMarkBadQuality(a); // to make sure markEntities does not complain! // balance the mesh to avoid empty parts midBalance(a); - print("--percent change in number of bad elements %f", + print("--percent change in number of bad elements %f", a->mesh->getPCU(), ((double) prev_count - (double) count) / (double) prev_count); iter++; } while(count < prev_count); double t1 = pcu::Time(); - print("bad shapes down from %d to %d in %f seconds", + print("bad shapes down from %d to %d in %f seconds", a->mesh->getPCU(), originalCount,count,t1-t0); } @@ -869,7 +869,7 @@ void alignElements(Adapt* a) } while(count < prev_count && i < max_iter); double t1 = pcu::Time(); - print("non-aligned elements down from %d to %d in %f seconds", + print("non-aligned elements down from %d to %d in %f seconds", a->mesh->getPCU(), originalCount,count,t1-t0); } @@ -878,7 +878,7 @@ void printQuality(Adapt* a) if ( ! a->input->shouldPrintQuality) return; double minqual = getMinQuality(a); - print("worst element quality is %e", minqual); + print("worst element quality is %e", a->mesh->getPCU(), minqual); } } diff --git a/ma/maSnap.cc b/ma/maSnap.cc index 5cafb87a6..b08e5cbae 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -914,7 +914,7 @@ void snap(Adapt* a) a->mesh->destroyTag(tag); double t1 = pcu::Time(); print("snapped in %f seconds: %ld targets, %ld non-layer snaps", - t1 - t0, targets, success); + a->mesh->getPCU(), t1 - t0, targets, success); if (a->hasLayer) checkLayerShape(a->mesh, "after snapping"); } diff --git a/ma/maTetrahedronize.cc b/ma/maTetrahedronize.cc index 332209a03..e0246daca 100644 --- a/ma/maTetrahedronize.cc +++ b/ma/maTetrahedronize.cc @@ -503,7 +503,7 @@ void tetrahedronize(Adapt* a) addAllLayerElements(r); tetrahedronizeCommon(r); double t1 = pcu::Time(); - print("boundary layer converted to tets in %f seconds",t1-t0); + print("boundary layer converted to tets in %f seconds", a->mesh->getPCU(), t1-t0); } /* start of the island pyramid cleanup system, @@ -675,14 +675,14 @@ void cleanupLayer(Adapt* a) double t0 = pcu::Time(); long n = prepareIslandCleanup(a); if (!n) { - print("no island pyramids found"); + print("no island pyramids found", a->mesh->getPCU()); return; } Refine* r = a->refine; addIslandPyramids(r); tetrahedronizeCommon(r); double t1 = pcu::Time(); - print("tetrahedronized %ld island pyramids in %f seconds", n, t1-t0); + print("tetrahedronized %ld island pyramids in %f seconds", a->mesh->getPCU(), n, t1-t0); } } diff --git a/test/repartition.cc b/test/repartition.cc index e43bbc6f3..387093b79 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -98,7 +98,6 @@ void balance(apf::Mesh2* m) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - //pcu::PCU *expanded_pcu_obj = new pcu::PCU(MPI_COMM_WORLD); auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); From 9075f4be713776432ad12dc0a25d9758416ad2d6 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 18 Mar 2024 12:35:13 -0400 Subject: [PATCH 072/141] ma folder using new pcu, all tests passing --- apf/apfNumbering.cc | 10 +++++++--- crv/crvAdapt.cc | 8 ++++---- crv/crvShape.cc | 18 +++++++++--------- ma/ma.cc | 18 +++++++++--------- ma/maAdapt.cc | 2 +- ma/maAdapt.h | 2 +- ma/maBalance.cc | 2 +- ma/maCoarsen.cc | 2 +- ma/maInput.cc | 8 ++++---- ma/maLayer.cc | 6 +++--- ma/maLayerCoarsen.cc | 2 +- ma/maLayerSnap.cc | 12 ++++++------ ma/maMatch.cc | 2 +- ma/maRefine.cc | 2 +- ma/maShape.cc | 14 +++++++------- ma/maSnap.cc | 4 ++-- ma/maTetrahedronize.cc | 6 +++--- test/mixedNumbering.cc | 2 ++ test/repartition.cc | 2 +- 19 files changed, 64 insertions(+), 58 deletions(-) diff --git a/apf/apfNumbering.cc b/apf/apfNumbering.cc index 6e0d78e30..2d7d408cc 100644 --- a/apf/apfNumbering.cc +++ b/apf/apfNumbering.cc @@ -227,10 +227,14 @@ void synchronize(Numbering * n, Sharing* shr, bool delete_shr) struct NoSharing : public Sharing { - int getOwner(MeshEntity*) {return PCU_Comm_Self();} + NoSharing(pcu::PCU *PCUObj):pcu_obj{PCUObj}{PCU_ALWAYS_ASSERT(PCUObj != nullptr);} + int getOwner(MeshEntity*) {return pcu_obj->Self();} bool isOwned(MeshEntity*) {return true;} virtual void getCopies(MeshEntity*, CopyArray&) {} bool isShared(MeshEntity*) {return false;} + + private: + pcu::PCU *pcu_obj; }; Numbering* numberNodes( @@ -277,7 +281,7 @@ Numbering* numberOwnedDimension(Mesh* mesh, const char* name, int dim, Numbering* numberOverlapDimension(Mesh* mesh, const char* name, int dim) { FieldShape* s = getConstant(dim); - Sharing* shr = new NoSharing(); + Sharing* shr = new NoSharing(mesh->getPCU()); return numberNodes(mesh, name, s, shr, true); } @@ -290,7 +294,7 @@ Numbering* numberOverlapNodes(Mesh* mesh, const char* name, FieldShape* s) { if (!s) s = mesh->getShape(); - Sharing* shr = new NoSharing(); + Sharing* shr = new NoSharing(mesh->getPCU()); return numberNodes(mesh, name, s, shr, true); } diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc index 43b85a184..edc2ff3d3 100644 --- a/crv/crvAdapt.cc +++ b/crv/crvAdapt.cc @@ -80,7 +80,7 @@ static void refine(ma::Adapt* a) } splitEdges(a); double t1 = pcu::Time(); - ma::print("split %li edges in %f seconds", a->mesh->getPCU(), count, t1-t0); + ma::print(a->mesh->getPCU(), "split %li edges in %f seconds", count, t1-t0); } int getValidityTag(ma::Mesh* m, ma::Entity* e, @@ -198,7 +198,7 @@ void adapt(ma::Input* in) fail("mesh must be bezier to adapt\n"); in->shapeHandler = crv::getShapeHandler; - ma::print("Curved Adaptation Version 2.0 !", in->mesh->getPCU()); + ma::print(in->mesh->getPCU(), "Curved Adaptation Version 2.0 !"); double t0 = pcu::Time(); ma::validateInput(in); Adapt* a = new Adapt(in); @@ -208,7 +208,7 @@ void adapt(ma::Input* in) for (int i=0; i < in->maximumIterations; ++i) { - ma::print("iteration %d", a->mesh->getPCU(), i); + ma::print(a->mesh->getPCU(), "iteration %d", i); ma::coarsen(a); ma::midBalance(a); crv::refine(a); @@ -228,7 +228,7 @@ void adapt(ma::Input* in) ma::printQuality(a); ma::postBalance(a); double t1 = pcu::Time(); - ma::print("mesh adapted in %f seconds", a->mesh->getPCU(), t1-t0); + ma::print(a->mesh->getPCU(), "mesh adapted in %f seconds", t1-t0); apf::printStats(a->mesh); crv::clearTags(a); delete a; diff --git a/crv/crvShape.cc b/crv/crvShape.cc index 08ee452a2..be35ea8da 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -714,8 +714,8 @@ int fixLargeBoundaryAngles(Adapt* a) } splitEdges(a); double t1 = pcu::Time(); - ma::print("split %d boundary edges with " - "large angles in %f seconds", a->mesh->getPCU(), count, t1-t0); + ma::print(a->mesh->getPCU(), "split %d boundary edges with " + "large angles in %f seconds", count, t1-t0); return 0; } @@ -734,8 +734,8 @@ static void collapseInvalidEdges(Adapt* a) } successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); - ma::print("Collapsed %d bad edges " - "in %f seconds", m->getPCU(), successCount, t1-t0); + ma::print(m->getPCU(), "Collapsed %d bad edges " + "in %f seconds", successCount, t1-t0); } static void swapInvalidEdges(Adapt* a) @@ -744,8 +744,8 @@ static void swapInvalidEdges(Adapt* a) EdgeSwapper es(a); ma::applyOperator(a,&es); double t1 = pcu::Time(); - ma::print("Swapped %d bad edges " - "in %f seconds", a->mesh->getPCU(), es.ns, t1-t0); + ma::print(a->mesh->getPCU(), "Swapped %d bad edges " + "in %f seconds", es.ns, t1-t0); } static void repositionInvalidEdges(Adapt* a) @@ -754,8 +754,8 @@ static void repositionInvalidEdges(Adapt* a) EdgeReshaper es(a); ma::applyOperator(a,&es); double t1 = pcu::Time(); - ma::print("Repositioned %d bad edges " - "in %f seconds", a->mesh->getPCU(), es.nr, t1-t0); + ma::print(a->mesh->getPCU(), "Repositioned %d bad edges " + "in %f seconds", es.nr, t1-t0); } int fixInvalidEdges(Adapt* a) @@ -850,7 +850,7 @@ void fixCrvElementShapes(Adapt* a) ++i; } while(count < prev_count && i < 6); // the second conditions is to make sure this does not take long double t1 = pcu::Time(); - ma::print("bad shapes down from %d to %d in %f seconds", a->mesh->getPCU(), + ma::print(a->mesh->getPCU(), "bad shapes down from %d to %d in %f seconds", originalCount,count,t1-t0); a->input->shouldForceAdaptation = false; } diff --git a/ma/ma.cc b/ma/ma.cc index 9ff1e14ed..72d137c67 100644 --- a/ma/ma.cc +++ b/ma/ma.cc @@ -26,10 +26,10 @@ void adapt(Input* in) validateInput(in); Adapt* a = new Adapt(in); preBalance(a); - print("version 2.0 !", a->mesh->getPCU()); + print(a->mesh->getPCU(), "version 2.0 !"); for (int i = 0; i < in->maximumIterations; ++i) { - print("iteration %d", a->mesh->getPCU(), i); + print(a->mesh->getPCU(), "iteration %d", i); coarsen(a); coarsenLayer(a); midBalance(a); @@ -51,7 +51,7 @@ void adapt(Input* in) delete in->solutionTransfer; delete in; double t1 = pcu::Time(); - print("mesh adapted in %f seconds", m->getPCU(), t1-t0); + print(m->getPCU(), "mesh adapted in %f seconds", t1-t0); apf::printStats(m); } @@ -66,10 +66,10 @@ void adaptVerbose(Input* in, bool verbose) validateInput(in); Adapt* a = new Adapt(in); preBalance(a); - print("version 2.0 - dev !", a->mesh->getPCU()); + print(a->mesh->getPCU(), "version 2.0 - dev !"); for (int i = 0; i < in->maximumIterations; ++i) { - print("iteration %d",a->mesh->getPCU(), i); + print(a->mesh->getPCU(), "iteration %d", i); coarsen(a); if (verbose && in->shouldCoarsen) ma_dbg::dumpMeshWithQualities(a,i,"after_coarsen"); @@ -99,14 +99,14 @@ void adaptVerbose(Input* in, bool verbose) */ int count = 0; double lMax = ma::getMaximumEdgeLength(a->mesh, a->sizeField); - print("Maximum (metric) edge length in the mesh is %f", a->mesh->getPCU(), lMax); + print(a->mesh->getPCU(), "Maximum (metric) edge length in the mesh is %f", lMax); while (lMax > 1.5) { - print("%dth additional refine-snap call", a->mesh->getPCU(), count); + print(a->mesh->getPCU(), "%dth additional refine-snap call", count); refine(a); snap(a); lMax = ma::getMaximumEdgeLength(a->mesh, a->sizeField); count++; - print("Maximum (metric) edge length in the mesh is %f", a->mesh->getPCU(), lMax); + print(a->mesh->getPCU(), "Maximum (metric) edge length in the mesh is %f", lMax); if (count > 5) break; } if (verbose) @@ -125,7 +125,7 @@ void adaptVerbose(Input* in, bool verbose) delete in->solutionTransfer; delete in; double t1 = pcu::Time(); - print("mesh adapted in %f seconds", m->getPCU(), t1-t0); + print(m->getPCU(), "mesh adapted in %f seconds", t1-t0); apf::printStats(m); } diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index 2a1feb2e7..b99bc7282 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -470,7 +470,7 @@ void clearBuildCallback(Adapt* a) a->buildCallback = 0; } -void print(const char* format, pcu::PCU *PCUObj, ...) +void print(pcu::PCU *PCUObj, const char* format, ...) { if (PCUObj->Self()) return; diff --git a/ma/maAdapt.h b/ma/maAdapt.h index 06ea608c7..0b7a0324d 100644 --- a/ma/maAdapt.h +++ b/ma/maAdapt.h @@ -164,7 +164,7 @@ Entity* rebuildElement( void setBuildCallback(Adapt* a, apf::BuildCallback* cb); void clearBuildCallback(Adapt* a); -void print(const char* format, pcu::PCU *PCUObj, ...) __attribute__((format(printf,1,3))); +void print(pcu::PCU *PCUObj, const char* format, ...) __attribute__((format(printf,2,3))); void setFlagOnClosure(Adapt* a, Entity* e, int flag); void syncFlag(Adapt* a, int dimension, int flag); diff --git a/ma/maBalance.cc b/ma/maBalance.cc index 5f4b10a5f..d402fe781 100644 --- a/ma/maBalance.cc +++ b/ma/maBalance.cc @@ -119,7 +119,7 @@ void printEntityImbalance(Mesh* m) double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); double p = (imbalance[m->getDimension()]-1)*100; - print("element imbalance %.0f%% of average", m->getPCU(), p); + print(m->getPCU(), "element imbalance %.0f%% of average", p); } double estimateWeightedImbalance(Adapt* a) diff --git a/ma/maCoarsen.cc b/ma/maCoarsen.cc index 4f7607901..7407f2493 100644 --- a/ma/maCoarsen.cc +++ b/ma/maCoarsen.cc @@ -249,7 +249,7 @@ bool coarsen(Adapt* a) } successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); - print("coarsened %li edges in %f seconds", m->getPCU(), successCount,t1-t0); + print(m->getPCU(), "coarsened %li edges in %f seconds", successCount,t1-t0); return true; } diff --git a/ma/maInput.cc b/ma/maInput.cc index 1c159ef4b..fb17deca7 100644 --- a/ma/maInput.cc +++ b/ma/maInput.cc @@ -154,14 +154,14 @@ static void updateMaxIterBasedOnSize(Mesh* m, Input* in) double maxMetricLength = getMaximumEdgeLength(m, in->sizeField); int iter = std::ceil(std::log2(maxMetricLength)); if (iter >= 10) { - print("ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations,\n" + print(m->getPCU(), "ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations,\n" " which is equal to or larger than the maximum of 10 allowed.\n" - " Setting the number of iteration to 10!", m->getPCU(), iter); + " Setting the number of iteration to 10!", iter); in->maximumIterations = 10; } else { - print("ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations.\n" - " Setting the number of iteration to %d!", m->getPCU(), iter, iter+1); + print(m->getPCU(), "ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations.\n" + " Setting the number of iteration to %d!", iter, iter+1); in->maximumIterations = iter+1; } } diff --git a/ma/maLayer.cc b/ma/maLayer.cc index 12814d44c..0f419ca2f 100644 --- a/ma/maLayer.cc +++ b/ma/maLayer.cc @@ -99,7 +99,7 @@ void resetLayer(Adapt* a) return; freezeLayer(a); double t1 = pcu::Time(); - print("marked %ld layer elements in %f seconds", a->mesh->getPCU(), n, t1 - t0); + print(a->mesh->getPCU(), "marked %ld layer elements in %f seconds", n, t1 - t0); } void findLayerBase(Adapt* a) @@ -148,7 +148,7 @@ void allowSplitInLayer(Adapt* a) if (getFlag(a,e,LAYER)) clearFlag(a,e,DONT_SPLIT); m->end(it); - print("allowing layer refinement", m->getPCU()); + print(m->getPCU(), "allowing layer refinement"); } void collectForLayerRefine(Refine* r) @@ -188,7 +188,7 @@ void checkLayerShape(Mesh* m, const char* key) m->end(it); n = m->getPCU()->Add(n); double t1 = pcu::Time(); - print("%s: checked layer quality in %f seconds: %ld unsafe elements", m->getPCU(), key, t1 - t0, n); + print(m->getPCU(), "%s: checked layer quality in %f seconds: %ld unsafe elements", key, t1 - t0, n); } } diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index 2c16491f0..150a0101c 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -283,7 +283,7 @@ bool coarsenLayer(Adapt* a) successCount += collapseAllStacks(a, d); } double t1 = pcu::Time(); - print("coarsened %li layer edges in %f seconds", m->getPCU(), successCount,t1-t0); + print(m->getPCU(), "coarsened %li layer edges in %f seconds", successCount,t1-t0); resetLayer(a); return true; } diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index ca8611908..63ecd4b80 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -274,7 +274,7 @@ static long snapAllCurves(Adapt* a, Tag* snapTag) LayerSnapper op(a, snapTag); crawlLayers(&op); double t1 = pcu::Time(); - print("snapped %ld curves in %f seconds", a->mesh->getPCU(), op.ncurves, t1 - t0); + print(a->mesh->getPCU(), "snapped %ld curves in %f seconds", op.ncurves, t1 - t0); return op.ncurves; } @@ -402,9 +402,9 @@ static bool checkForUnsnap(Adapt* a, Tag* snapTag) bool notOk = a->mesh->getPCU()->Or(op.foundAnything); double t1 = pcu::Time(); if (notOk) - print("checked snapped curves in %f seconds, found some to unsnap", a->mesh->getPCU(), t1 - t0); + print(a->mesh->getPCU(), "checked snapped curves in %f seconds, found some to unsnap", t1 - t0); else - print("checked snapped curves in %f seconds, all ok", a->mesh->getPCU(), t1 - t0); + print(a->mesh->getPCU(), "checked snapped curves in %f seconds, all ok", t1 - t0); return notOk; } @@ -437,7 +437,7 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) PCU_ALWAYS_ASSERT(m->hasTag(v, snapTag)); } n = m->getPCU()->Add(n); - print("fed back unsnap flag from %ld tops", m->getPCU(), n); + print(m->getPCU(), "fed back unsnap flag from %ld tops", n); } /* for each layer curve whose base vertex @@ -528,7 +528,7 @@ static long unsnapMarkedCurves(Adapt* a, Tag* snapTag) Unsnapper op(a, snapTag); crawlLayers(&op); double t1 = pcu::Time(); - print("unsnapped %ld curves in %f seconds", a->mesh->getPCU(), op.ncurves, t1 - t0); + print(a->mesh->getPCU(), "unsnapped %ld curves in %f seconds", op.ncurves, t1 - t0); return op.ncurves; } @@ -550,7 +550,7 @@ void snapLayer(Adapt* a, Tag* snapTag) } delete l; double t1 = pcu::Time(); - print("finished snapping %ld of %ld layer curves in %f seconds", a->mesh->getPCU(), + print(a->mesh->getPCU(), "finished snapping %ld of %ld layer curves in %f seconds", nsnapped - nunsnapped, nsnapped, t1 - t0); } diff --git a/ma/maMatch.cc b/ma/maMatch.cc index 020daca43..6cc1c9474 100644 --- a/ma/maMatch.cc +++ b/ma/maMatch.cc @@ -72,7 +72,7 @@ void matchNewElements(Refine* r) } } face_count = m->getPCU()->Add(face_count); - print("updated matching for %li faces", m->getPCU(), face_count); + print(m->getPCU(), "updated matching for %li faces", face_count); } void preventMatchedCavityMods(Adapt* a) diff --git a/ma/maRefine.cc b/ma/maRefine.cc index 661377d59..b437e45c5 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -438,7 +438,7 @@ bool refine(Adapt* a) destroySplitElements(r); forgetNewEntities(r); double t1 = pcu::Time(); - print("refined %li edges in %f seconds",a->mesh->getPCU(),count,t1-t0); + print(a->mesh->getPCU(),"refined %li edges in %f seconds",count,t1-t0); resetLayer(a); if (a->hasLayer) checkLayerShape(a->mesh, "after refinement"); diff --git a/ma/maShape.cc b/ma/maShape.cc index 845bf21ac..67812eead 100644 --- a/ma/maShape.cc +++ b/ma/maShape.cc @@ -820,7 +820,7 @@ void fixElementShapes(Adapt* a) if ( ! count) break; prev_count = count; - print("--iter %d of shape correction loop: #bad elements %d", a->mesh->getPCU(), iter, count); + print(a->mesh->getPCU(), "--iter %d of shape correction loop: #bad elements %d", iter, count); time = fixLargeAngles(a); /* We need to snap the new verts as soon as they are * created (to avoid future problems). At the moment @@ -830,20 +830,20 @@ void fixElementShapes(Adapt* a) if (a->mesh->getDimension() == 3) snap(a); count = markBadQuality(a); - print("--fixLargeAngles in %f seconds: #bad elements %d", a->mesh->getPCU(), time,count); + print(a->mesh->getPCU(), "--fixLargeAngles in %f seconds: #bad elements %d", time,count); time = fixShortEdgeElements(a); count = markBadQuality(a); - print("--fixShortEdgeElements in %f seconds: #bad elements %d", a->mesh->getPCU(), time,count); + print(a->mesh->getPCU(), "--fixShortEdgeElements in %f seconds: #bad elements %d", time,count); if (count >= prev_count) unMarkBadQuality(a); // to make sure markEntities does not complain! // balance the mesh to avoid empty parts midBalance(a); - print("--percent change in number of bad elements %f", a->mesh->getPCU(), + print(a->mesh->getPCU(), "--percent change in number of bad elements %f", ((double) prev_count - (double) count) / (double) prev_count); iter++; } while(count < prev_count); double t1 = pcu::Time(); - print("bad shapes down from %d to %d in %f seconds", a->mesh->getPCU(), + print(a->mesh->getPCU(), "bad shapes down from %d to %d in %f seconds", originalCount,count,t1-t0); } @@ -869,7 +869,7 @@ void alignElements(Adapt* a) } while(count < prev_count && i < max_iter); double t1 = pcu::Time(); - print("non-aligned elements down from %d to %d in %f seconds", a->mesh->getPCU(), + print(a->mesh->getPCU(), "non-aligned elements down from %d to %d in %f seconds", originalCount,count,t1-t0); } @@ -878,7 +878,7 @@ void printQuality(Adapt* a) if ( ! a->input->shouldPrintQuality) return; double minqual = getMinQuality(a); - print("worst element quality is %e", a->mesh->getPCU(), minqual); + print(a->mesh->getPCU(), "worst element quality is %e", minqual); } } diff --git a/ma/maSnap.cc b/ma/maSnap.cc index b08e5cbae..feb55ab43 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -913,8 +913,8 @@ void snap(Adapt* a) apf::removeTagFromDimension(a->mesh, tag, 0); a->mesh->destroyTag(tag); double t1 = pcu::Time(); - print("snapped in %f seconds: %ld targets, %ld non-layer snaps", - a->mesh->getPCU(), t1 - t0, targets, success); + print(a->mesh->getPCU(), "snapped in %f seconds: %ld targets, %ld non-layer snaps", + t1 - t0, targets, success); if (a->hasLayer) checkLayerShape(a->mesh, "after snapping"); } diff --git a/ma/maTetrahedronize.cc b/ma/maTetrahedronize.cc index e0246daca..fd2cb1e11 100644 --- a/ma/maTetrahedronize.cc +++ b/ma/maTetrahedronize.cc @@ -503,7 +503,7 @@ void tetrahedronize(Adapt* a) addAllLayerElements(r); tetrahedronizeCommon(r); double t1 = pcu::Time(); - print("boundary layer converted to tets in %f seconds", a->mesh->getPCU(), t1-t0); + print(a->mesh->getPCU(), "boundary layer converted to tets in %f seconds", t1-t0); } /* start of the island pyramid cleanup system, @@ -675,14 +675,14 @@ void cleanupLayer(Adapt* a) double t0 = pcu::Time(); long n = prepareIslandCleanup(a); if (!n) { - print("no island pyramids found", a->mesh->getPCU()); + print(a->mesh->getPCU(), "no island pyramids found"); return; } Refine* r = a->refine; addIslandPyramids(r); tetrahedronizeCommon(r); double t1 = pcu::Time(); - print("tetrahedronized %ld island pyramids in %f seconds", a->mesh->getPCU(), n, t1-t0); + print(a->mesh->getPCU(), "tetrahedronized %ld island pyramids in %f seconds", n, t1-t0); } } diff --git a/test/mixedNumbering.cc b/test/mixedNumbering.cc index 3613eb5e8..6a0dbf918 100644 --- a/test/mixedNumbering.cc +++ b/test/mixedNumbering.cc @@ -50,6 +50,7 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); + { auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); @@ -59,5 +60,6 @@ int main(int argc, char** argv) write_output(m, argv[3]); m->destroyNative(); apf::destroyMesh(m); + } MPI_Finalize(); } diff --git a/test/repartition.cc b/test/repartition.cc index 387093b79..c8174cbad 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -32,7 +32,7 @@ void freeMesh(apf::Mesh* m) CreateGroupCommResult createGroupComm(pcu::PCU *PCUObj) { apf::Contract contract(inputPartCount, PCUObj->Peers()); - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); PCU_ALWAYS_ASSERT(self == PCUObj->Self()); int group; int groupRank; From efc2d775e9c3bd9de3e4f1ed652e583fa528bd60 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 20 Mar 2024 15:20:23 -0400 Subject: [PATCH 073/141] changed phastaio_initStats to not check if PCU is intitialized. Also using PCUHandle and PCU2 in phiotimer.h --- phasta/phiotimer.cc | 83 ++++++++++++++++++++++----------------------- phasta/phiotimer.h | 8 +++-- 2 files changed, 47 insertions(+), 44 deletions(-) diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index ecc1f7b40..f1dc76a60 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -1,7 +1,7 @@ #include #include +#include #include -#include #include #include @@ -32,7 +32,7 @@ void phastaio_time(phastaioTime* t) { *t = _rdtsc(); //intel intrinsic } /* determine the reference clock frequency */ -static size_t phastaio_getCyclesPerMicroSec() { +static size_t phastaio_getCyclesPerMicroSec(PCUHandle h) { const size_t usec = 5*MILLION; size_t cpus, cycles; phastaioTime t0, t1; @@ -44,7 +44,7 @@ static size_t phastaio_getCyclesPerMicroSec() { phastaio_time(&t1); cycles = t1 - t0; cpus = ((double)cycles)/(usec); - if(!PCU_Comm_Self()) { + if(!PCU_Comm_Self2(h)) { std::stringstream ss; ss << "cycles " << cycles << " us " << usec << " cycles per micro second " << cpus << "\n"; @@ -61,7 +61,7 @@ size_t phastaio_time_diff(phastaioTime* start, phastaioTime* end) { } #elif defined(USE_PCU_TIME) void phastaio_time(phastaioTime* t) { - *t = PCU_Time(); + *t = pcu::Time(); } /*return elapsed time in micro seconds*/ size_t phastaio_time_diff(phastaioTime* start, phastaioTime* end) { @@ -140,12 +140,12 @@ static const char* getFileName() { return names[phastaio_global_stats.fileIdx]; } -static void printMinMaxAvgSzt(const char* key, size_t v) { - size_t min = PCU_Min_SizeT(v); - size_t max = PCU_Max_SizeT(v); - size_t tot = PCU_Add_SizeT(v); - double avg = ((double)tot)/PCU_Comm_Peers(); - if(!PCU_Comm_Self()) { +static void printMinMaxAvgSzt(const char* key, size_t v, PCUHandle h) { + size_t min = PCU_Min_SizeT2(h, v); + size_t max = PCU_Max_SizeT2(h, v); + size_t tot = PCU_Add_SizeT2(h, v); + double avg = ((double)tot)/PCU_Comm_Peers2(h); + if(!PCU_Comm_Self2(h)) { std::stringstream ss; ss << getFileName() << "_" << key << "min max avg" << min << " " << max << " " << avg << "\n"; @@ -154,12 +154,12 @@ static void printMinMaxAvgSzt(const char* key, size_t v) { } } -static void printMinMaxAvgDbl(const char* key, double v) { - double min = PCU_Min_Double(v); - double max = PCU_Max_Double(v); - double tot = PCU_Add_Double(v); - double avg = tot/PCU_Comm_Peers(); - if(!PCU_Comm_Self()) +static void printMinMaxAvgDbl(const char* key, double v, PCUHandle h) { + double min = PCU_Min_Double2(h, v); + double max = PCU_Max_Double2(h, v); + double tot = PCU_Add_Double2(h, v); + double avg = tot/PCU_Comm_Peers2(h); + if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s_%s min max avg %f %f %f\n", getFileName(), key, min, max, avg); } @@ -214,8 +214,8 @@ static size_t phastaio_getCloseTime() { return phastaio_global_stats.closeTime[i]; } -void phastaio_printStats() { - if(!PCU_Comm_Self()) { +void phastaio_printStats(PCUHandle h) { + if(!PCU_Comm_Self2(h)) { const size_t us = 1000; phastaioTime t0,t1; size_t elapsed; @@ -232,58 +232,57 @@ void phastaio_printStats() { size_t totalus = 0; size_t totalbytes = 0; phastaio_setfile(chefFile); - if(!PCU_Comm_Self()) + if(!PCU_Comm_Self2(h)) lion_eprint(1, "phastaio_filename %s\n", getFileName()); - int reads = PCU_Max_Int((int)phastaio_getReads()); + int reads = PCU_Max_Int2(h, (int)phastaio_getReads()); if(reads) { totalus += phastaio_getReadTime(); totalbytes += phastaio_getReadBytes(); - printMinMaxAvgSzt("reads", phastaio_getReads()); - printMinMaxAvgSzt("readTime (us)", phastaio_getReadTime()); - printMinMaxAvgSzt("readBytes (B)", phastaio_getReadBytes()); + printMinMaxAvgSzt("reads", phastaio_getReads(), h); + printMinMaxAvgSzt("readTime (us)", phastaio_getReadTime(), h); + printMinMaxAvgSzt("readBytes (B)", phastaio_getReadBytes(), h); double bw = ((double)phastaio_getReadBytes())/phastaio_getReadTime(); - printMinMaxAvgDbl("readBandwidth (MB/s)", bw); + printMinMaxAvgDbl("readBandwidth (MB/s)", bw, h); /* B * 10^6us * 1MB = MB * - ------ ----- -- * us 1s 10^6B s */ } - int writes = PCU_Max_Int((int)phastaio_getWrites()); + int writes = PCU_Max_Int2(h, (int)phastaio_getWrites()); if(writes) { totalus += phastaio_getWriteTime(); totalbytes += phastaio_getWriteBytes(); - printMinMaxAvgSzt("writes", phastaio_getWrites()); - printMinMaxAvgSzt("writeTime (us)", phastaio_getWriteTime()); - printMinMaxAvgSzt("writeBytes (B)", phastaio_getWriteBytes()); + printMinMaxAvgSzt("writes", phastaio_getWrites(), h); + printMinMaxAvgSzt("writeTime (us)", phastaio_getWriteTime(), h); + printMinMaxAvgSzt("writeBytes (B)", phastaio_getWriteBytes(), h); printMinMaxAvgDbl("writeBandwidth (MB/s)", - ((double)phastaio_getWriteBytes())/phastaio_getWriteTime()); + ((double)phastaio_getWriteBytes())/phastaio_getWriteTime(), h); } - int opens = PCU_Max_Int((int)phastaio_getOpens()); + int opens = PCU_Max_Int2(h, (int)phastaio_getOpens()); if(opens) { totalus += phastaio_getOpenTime(); - printMinMaxAvgSzt("opens", phastaio_getOpens()); - printMinMaxAvgSzt("openTime (us)", phastaio_getOpenTime()); + printMinMaxAvgSzt("opens", phastaio_getOpens(), h); + printMinMaxAvgSzt("openTime (us)", phastaio_getOpenTime(), h); } - int closes = PCU_Max_Int((int)phastaio_getCloses()); + int closes = PCU_Max_Int2(h, (int)phastaio_getCloses()); if(closes) { totalus += phastaio_getCloseTime(); - printMinMaxAvgSzt("closes", phastaio_getCloses()); - printMinMaxAvgSzt("closeTime (us)", phastaio_getCloseTime()); + printMinMaxAvgSzt("closes", phastaio_getCloses(), h); + printMinMaxAvgSzt("closeTime (us)", phastaio_getCloseTime(), h); } if(totalbytes) { - printMinMaxAvgSzt("totalTime (us)", totalus); - printMinMaxAvgSzt("totalBytes (B)", totalbytes); + printMinMaxAvgSzt("totalTime (us)", totalus, h); + printMinMaxAvgSzt("totalBytes (B)", totalbytes, h); printMinMaxAvgDbl("effectiveBandwidth (MB/s)", - ((double)totalbytes)/totalus); + ((double)totalbytes)/totalus, h); } } } -void phastaio_initStats() { - if( !PCU_Comm_Initialized() ) - PCU_Comm_Init(); +void phastaio_initStats(PCUHandle h) { + //removed checking if PCU is initiallized as it's a class invarient of apf::Mesh that it is #ifdef __INTEL_COMPILER - phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(); + phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(h); #endif for(int i=0; i + #ifdef __cplusplus extern "C" { #endif +typedef struct PCUHandle PCUHandle; + #define PHASTAIO_READTIME(cmd,bytes) {\ phastaioTime t0,t1;\ phastaio_time(&t0);\ @@ -81,9 +85,9 @@ void phastaio_addOpenTime(size_t t); /* \brief accumulate time closing */ void phastaio_addCloseTime(size_t t); /* \brief initialize the counters and timers */ -void phastaio_initStats(); +void phastaio_initStats(PCUHandle h); /* \brief print io information */ -void phastaio_printStats(); +void phastaio_printStats(PCUHandle h); /* \brief set the current file to record counters and timers for * \detail see phastaio_file enum */ void phastaio_setfile(int f); From d4249631af1b46b6da38f7dd6dc2b10a6a2c2c5e Mon Sep 17 00:00:00 2001 From: flagdanger Date: Sun, 24 Mar 2024 21:40:02 -0400 Subject: [PATCH 074/141] Phasta folder changed to use new PCU, passes tests. --- phasta/adaptLvlSet_loop.cc | 30 ++++----- phasta/chef.cc | 15 +++-- phasta/chef.h | 10 +-- phasta/chefStream.cc | 26 ++++---- phasta/condense.cc | 23 +++---- phasta/cut_interface.cc | 11 ++-- phasta/migrate_interface.cc | 15 +++-- phasta/ph.cc | 39 ++++++----- phasta/ph.h | 12 ++-- phasta/phAdapt.cc | 31 +++++---- phasta/phAxisymmetry.cc | 13 ++-- phasta/phBC.cc | 10 +-- phasta/phBC.h | 5 +- phasta/phBlock.cc | 1 - phasta/phBubble.cc | 11 ++-- phasta/phCook.cc | 114 +++++++++++++++------------------ phasta/phFilterMatching.cc | 17 +++-- phasta/phGeomBC.cc | 7 +- phasta/phGrowthCurves.cc | 1 - phasta/phGrowthCurves_empty.cc | 1 - phasta/phIO.c | 22 ++++--- phasta/phInput.cc | 14 ++-- phasta/phInput.h | 8 ++- phasta/phInterfaceCutter.cc | 1 - phasta/phLinks.cc | 1 - phasta/phMeshQuality.cc | 1 - phasta/phOutput.cc | 7 +- phasta/phPartition.cc | 1 - phasta/phRestart.cc | 19 +++--- phasta/ph_convert.cc | 58 ++++++++--------- phasta/phiotimer.cc | 1 + phasta/phstream.cc | 72 ++++++++++----------- phasta/phstream.h | 23 +++---- phasta/readUrPrep.cc | 16 +++-- phasta/threshold.cc | 9 +-- 35 files changed, 319 insertions(+), 326 deletions(-) diff --git a/phasta/adaptLvlSet_loop.cc b/phasta/adaptLvlSet_loop.cc index df1fc3727..08832a371 100644 --- a/phasta/adaptLvlSet_loop.cc +++ b/phasta/adaptLvlSet_loop.cc @@ -10,8 +10,6 @@ generates PHASTA files containing the mesh and field information. */ - -#include #include #include #include @@ -21,6 +19,7 @@ #endif #include #include +#include namespace { void freeMesh(apf::Mesh* m) { @@ -39,8 +38,9 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); - PCU_Protect(); + { + auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -49,28 +49,28 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(); + GRStream* grs = makeGRStream(expanded_pcu_obj.get()); ph::Input ctrl; - ctrl.load("adaptLvlSet.inp"); + ctrl.load("adaptLvlSet.inp", expanded_pcu_obj.get()); //preprocess (define bubbles) - chef::cook(g,m,ctrl,grs); - RStream* rs = makeRStream(); - attachRStream(grs,rs); + chef::cook(g,m,ctrl,grs,expanded_pcu_obj.get()); + RStream* rs = makeRStream(expanded_pcu_obj.get()); + attachRStream(grs,rs,expanded_pcu_obj.get()); reconfigureChef(ctrl); // attach the solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs,grs); - attachRStream(grs,rs); + chef::cook(g,m,ctrl,rs,grs,expanded_pcu_obj.get()); + attachRStream(grs,rs,expanded_pcu_obj.get()); // attach a solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs); - destroyGRStream(grs); - destroyRStream(rs); + chef::cook(g,m,ctrl,rs,expanded_pcu_obj.get()); + destroyGRStream(grs,expanded_pcu_obj.get()); + destroyRStream(rs,expanded_pcu_obj.get()); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/chef.cc b/phasta/chef.cc index c946cab2d..40b03453e 100644 --- a/phasta/chef.cc +++ b/phasta/chef.cc @@ -1,9 +1,9 @@ #include #include #include -#include #include #include +#include #ifdef HAVE_SIMMETRIX #include #include @@ -31,10 +31,11 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - PCU_Protect(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::Protect(); lion_set_verbosity(1); - if( !PCU_Comm_Self() ) { + if( !PCUObj.get()->Self() ) { lion_oprint(1,"PUMI Git hash %s\n", pumi_version()); lion_oprint(1,"PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); } @@ -55,8 +56,8 @@ int main(int argc, char** argv) std::string inputPath = "adapt.inp"; if(argc==2) inputPath = argv[1]; ph::Input in; - in.load(inputPath.c_str()); - chef::cook(g,m,in); + in.load(inputPath.c_str(), PCUObj.get()); + chef::cook(g,m,in,PCUObj.get()); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -68,7 +69,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/chef.h b/phasta/chef.h index c58a6ab3e..d60c0bb32 100644 --- a/phasta/chef.h +++ b/phasta/chef.h @@ -13,18 +13,18 @@ struct RStream; struct GRStream; namespace chef { /** @brief read and write to and from files */ - void cook(gmi_model*& g, apf::Mesh2*& m); + void cook(gmi_model*& g, apf::Mesh2*& m, pcu::PCU *PCUObj); /** @brief read and write to and from files and load control info*/ - void cook(gmi_model*& g, apf::Mesh2*& m, ph::Input& ctrl); + void cook(gmi_model*& g, apf::Mesh2*& m, ph::Input& ctrl, pcu::PCU *PCUObj); /** @brief read from stream and write to files */ void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* in); + ph::Input& ctrl, RStream* in, pcu::PCU *PCUObj); /** @brief read from files and write to stream */ void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, GRStream* out); + ph::Input& ctrl, GRStream* out, pcu::PCU *PCUObj); /** @brief read and write to and from streams */ void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* in, GRStream* out); + ph::Input& ctrl, RStream* in, GRStream* out, pcu::PCU *PCUObj); /** @brief extract a field from a packed field */ apf::Field* extractField(apf::Mesh* m, const char* packedFieldname, diff --git a/phasta/chefStream.cc b/phasta/chefStream.cc index 79ff422f7..11f6d2d08 100644 --- a/phasta/chefStream.cc +++ b/phasta/chefStream.cc @@ -10,8 +10,6 @@ generates PHASTA files containing the mesh and field information. */ - -#include #include #include #include @@ -20,6 +18,7 @@ #include #endif #include +#include namespace { void freeMesh(apf::Mesh* m) { @@ -35,8 +34,9 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); - PCU_Protect(); + { + auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -45,21 +45,21 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(); + GRStream* grs = makeGRStream(expanded_pcu_obj.get()); ph::Input ctrl; - ctrl.load("adapt.inp"); - chef::cook(g,m,ctrl,grs); - RStream* rs = makeRStream(); - attachRStream(grs,rs); + ctrl.load("adapt.inp", expanded_pcu_obj.get()); + chef::cook(g,m,ctrl,grs,expanded_pcu_obj.get()); + RStream* rs = makeRStream(expanded_pcu_obj.get()); + attachRStream(grs,rs,expanded_pcu_obj.get()); reconfigureChef(ctrl); - chef::cook(g,m,ctrl,rs); - destroyGRStream(grs); - destroyRStream(rs); + chef::cook(g,m,ctrl,rs,expanded_pcu_obj.get()); + destroyGRStream(grs,expanded_pcu_obj.get()); + destroyRStream(rs,expanded_pcu_obj.get()); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/condense.cc b/phasta/condense.cc index 14dfc90d3..21e5e95d5 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -12,9 +11,10 @@ #include #include #include +#include namespace { - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { return fopen(path, "r"); } @@ -27,24 +27,25 @@ namespace { } }; - void checkInputs(int argc, char** argv) { + void checkInputs(int argc, char** argv, pcu::PCU *pcu_obj) { if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj->Self() ) lion_oprint(1,"Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } - if ( !PCU_Comm_Self() ) + if ( !pcu_obj->Self() ) lion_oprint(1,"Input control file %s reduction factor %s\n", argv[1], argv[2]); } } int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - PCU_Protect(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::Protect(); lion_set_verbosity(1); - checkInputs(argc,argv); + checkInputs(argc,argv,pcu_obj.get()); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -53,9 +54,9 @@ int main(int argc, char** argv) { gmi_register_mesh(); GroupCode code; ph::Input in; - in.load(argv[1]); + in.load(argv[1], pcu_obj.get()); in.openfile_read = openfile_read; - code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str()); + code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str(), pcu_obj.get()); chef::readAndAttachFields(in,code.mesh); apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); code.ctrl = in; @@ -64,6 +65,6 @@ int main(int argc, char** argv) { gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/cut_interface.cc b/phasta/cut_interface.cc index d600211e5..09e30d713 100644 --- a/phasta/cut_interface.cc +++ b/phasta/cut_interface.cc @@ -1,7 +1,6 @@ #include "ph.h" #include "phInterfaceCutter.h" #include "phAttrib.h" -#include #include #ifdef HAVE_SIMMETRIX #include @@ -15,6 +14,7 @@ #include #include #include +#include char const* modelfile; char const* attribfile; @@ -32,8 +32,9 @@ int main(int argc, char** argv) lion_eprint(1," to take combined model and attributes file (by simTranslate)\n"); return 0; } - PCU_Comm_Init(); - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() == 1); + { + auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + PCU_ALWAYS_ASSERT(expanded_pcu_obj.get()->Peers() == 1); #ifdef HAVE_SIMMETRIX SimModel_start(); Sim_readLicenseFile(0); @@ -60,7 +61,7 @@ int main(int argc, char** argv) gm = gmi_sim_load(modelfile, attribfile); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); - apf::Mesh2* m = ph::loadMesh(gm, meshfile); + apf::Mesh2* m = ph::loadMesh(gm, meshfile, expanded_pcu_obj.get()); m->verify(); #ifdef HAVE_SIMMETRIX if (ph::mesh_has_ext(meshfile, "sms")) @@ -81,6 +82,6 @@ int main(int argc, char** argv) SimModel_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/migrate_interface.cc b/phasta/migrate_interface.cc index 804e9098c..5f907f238 100644 --- a/phasta/migrate_interface.cc +++ b/phasta/migrate_interface.cc @@ -2,7 +2,6 @@ #include "phBC.h" #include "phInterfaceCutter.h" #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include #include +#include namespace { @@ -31,10 +31,10 @@ void freeMesh(apf::Mesh* m) apf::destroyMesh(m); } -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { if (argc < 4 || argc > 5) { - if ( !PCU_Comm_Self() ) { + if ( !pcu_obj->Self() ) { lion_eprint(1,"Usage: %s \n", argv[0]); lion_eprint(1," to take model and attributes in separate files\n"); lion_eprint(1,"Usage: %s \n", argv[0]); @@ -58,7 +58,8 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -68,11 +69,11 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,pcu_obj.get()); gmi_model* gm; gm = gmi_sim_load(modelFile, attribFile); - apf::Mesh2* m = apf::loadMdsMesh(gm, inMesh); + apf::Mesh2* m = apf::loadMdsMesh(gm, inMesh, pcu_obj.get()); m->verify(); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); @@ -89,6 +90,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/ph.cc b/phasta/ph.cc index 17b404120..e1fcccf21 100644 --- a/phasta/ph.cc +++ b/phasta/ph.cc @@ -1,4 +1,3 @@ -#include #include #include "ph.h" @@ -63,14 +62,14 @@ void goToParentDir() { my_chdir(".."); } -void goToStepDir(int step, bool all_mkdir) +void goToStepDir(int step, pcu::PCU *PCUObj, bool all_mkdir) { std::stringstream ss; ss << step; std::string s = ss.str(); - if (all_mkdir || !PCU_Comm_Self()) + if (all_mkdir || !PCUObj->Self()) my_mkdir(s.c_str()); - PCU_Barrier(); + PCUObj->Barrier(); my_chdir(s.c_str()); } @@ -78,22 +77,22 @@ enum { DIR_FANOUT = 2048 }; -std::string setupOutputDir(bool all_mkdir) +std::string setupOutputDir(pcu::PCU *PCUObj, bool all_mkdir) { std::stringstream ss; - ss << PCU_Comm_Peers() << "-procs_case/"; + ss << PCUObj->Peers() << "-procs_case/"; std::string s = ss.str(); - if (all_mkdir || !PCU_Comm_Self()) + if (all_mkdir || !PCUObj->Self()) my_mkdir(s.c_str()); - PCU_Barrier(); + PCUObj->Barrier(); return s; } -void setupOutputSubdir(std::string& path, bool all_mkdir) +void setupOutputSubdir(std::string& path, pcu::PCU *PCUObj, bool all_mkdir) { - if (PCU_Comm_Peers() <= DIR_FANOUT) + if (PCUObj->Peers() <= DIR_FANOUT) return; - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int subSelf = self % DIR_FANOUT; int subGroup = self / DIR_FANOUT; std::stringstream ss; @@ -101,14 +100,14 @@ void setupOutputSubdir(std::string& path, bool all_mkdir) path = ss.str(); if (all_mkdir || !subSelf) my_mkdir(path.c_str()); - PCU_Barrier(); + PCUObj->Barrier(); } -void setupInputSubdir(std::string& path) +void setupInputSubdir(std::string& path, pcu::PCU *PCUObj) { - if (PCU_Comm_Peers() <= DIR_FANOUT) + if (PCUObj->Peers() <= DIR_FANOUT) return; - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int subGroup = self / DIR_FANOUT; std::string newpath; std::stringstream ss; @@ -127,16 +126,16 @@ void setupInputSubdir(std::string& path) // lion_oprint(1,"Rank: %d - Path in setupInputSubdir: %s\n", self, ss.str().c_str()); path = ss.str(); - PCU_Barrier(); + PCUObj->Barrier(); } -void writeAuxiliaryFiles(std::string path, int timestep_or_dat) +void writeAuxiliaryFiles(std::string path, int timestep_or_dat, pcu::PCU *PCUObj) { std::string numpePath = path; numpePath += "numpe.in"; std::ofstream numpe(numpePath.c_str()); PCU_ALWAYS_ASSERT(numpe.is_open()); - numpe << PCU_Comm_Peers() << '\n'; + numpe << PCUObj->Peers() << '\n'; numpe.close(); std::string numstartPath = path; numstartPath += "numstart.dat"; @@ -157,7 +156,7 @@ bool mesh_has_ext(const char* filename, const char* ext) } } -apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile) { +apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj) { apf::Mesh2* mesh; #ifdef HAVE_SIMMETRIX /* if it is a simmetrix mesh */ @@ -174,7 +173,7 @@ apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile) { #endif /* if it is a SCOREC mesh */ { - mesh = apf::loadMdsMesh(g, meshfile); + mesh = apf::loadMdsMesh(g, meshfile, PCUObj); } return mesh; } diff --git a/phasta/ph.h b/phasta/ph.h index c3ab09508..e5b8b9dc9 100644 --- a/phasta/ph.h +++ b/phasta/ph.h @@ -9,14 +9,14 @@ namespace ph { void fail(const char* format, ...) __attribute__((noreturn,format(printf,1,2))); -void goToStepDir(int step, bool all_mkdir=false); +void goToStepDir(int step, pcu::PCU *PCUObj, bool all_mkdir=false); void goToParentDir(); -std::string setupOutputDir(bool all_mkdir=false); -void setupInputSubdir(std::string& path); -void setupOutputSubdir(std::string& path, bool all_mkdir=false); -void writeAuxiliaryFiles(std::string path, int timestep_or_dat); +std::string setupOutputDir(pcu::PCU *PCUObj, bool all_mkdir=false); +void setupInputSubdir(std::string& path, pcu::PCU *PCUObj); +void setupOutputSubdir(std::string& path, pcu::PCU *PCUObj, bool all_mkdir=false); +void writeAuxiliaryFiles(std::string path, int timestep_or_dat, pcu::PCU *PCUObj); bool mesh_has_ext(const char* filename, const char* ext); -apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile); +apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj = nullptr); } diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index 05ee264be..a40ada8c4 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -2,7 +2,6 @@ #include "chef.h" #include "ph.h" #include -#include #include #include #include @@ -14,7 +13,7 @@ namespace ph { void setupBalance(const char* key, std::string& method, - bool& parmaBal, bool& zoltanBal, bool& zoltanRibBal) { + bool& parmaBal, bool& zoltanBal, bool& zoltanRibBal, pcu::PCU *PCUObj) { if ( method == "parma" ) { parmaBal = true; zoltanBal = false; @@ -32,15 +31,15 @@ void setupBalance(const char* key, std::string& method, zoltanBal = false; zoltanRibBal = false; } else { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_eprint(1, "warning: ignoring unknown value of %s = %s\n", key, method.c_str()); } } -void setupMatching(ma::Input& in) { - if (!PCU_Comm_Self()) +void setupMatching(ma::Input& in, pcu::PCU *PCUObj) { + if (!PCUObj->Self()) lion_oprint(1,"Matched mesh: disabling" " snapping, and shape correction,\n"); in.shouldSnap = false; @@ -66,13 +65,13 @@ struct AdaptCallback : public Parma_GroupCode //override with user inputs if specified setupBalance("preAdaptBalanceMethod", in->preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, mesh->getPCU()); bool ignored; setupBalance("midAdaptBalanceMethod", in->midAdaptBalanceMethod, - ma_in->shouldRunMidParma, ma_in->shouldRunMidZoltan, ignored); + ma_in->shouldRunMidParma, ma_in->shouldRunMidZoltan, ignored, mesh->getPCU()); setupBalance("postAdaptBalanceMethod", in->postAdaptBalanceMethod, ma_in->shouldRunPostParma, ma_in->shouldRunPostZoltan, - ma_in->shouldRunPostZoltanRib); + ma_in->shouldRunPostZoltanRib, mesh->getPCU()); ma_in->shouldTransferParametric = in->transferParametric; ma_in->shouldSnap = in->snap; ma_in->maximumIterations = in->maxAdaptIterations; @@ -88,7 +87,7 @@ struct AdaptCallback : public Parma_GroupCode ma_in->shouldRunPreZoltan = true; } if (mesh->hasMatching()) - ph::setupMatching(*ma_in); + ph::setupMatching(*ma_in, mesh->getPCU()); ma::adapt(ma_in); } }; @@ -112,9 +111,9 @@ static int getShrinkFactor(apf::Mesh* m, double minPartDensity) { return factor; } -static void warnAboutShrinking(int factor) { - int nprocs = PCU_Comm_Peers() / factor; - if (!PCU_Comm_Self()) { +static void warnAboutShrinking(int factor, pcu::PCU *PCUObj) { + int nprocs = PCUObj->Peers() / factor; + if (!PCUObj->Self()) { lion_eprint(1,"sensing mesh is spread too thin: " "adapting with %d procs\n", nprocs); } @@ -128,7 +127,7 @@ void adaptShrunken(apf::Mesh2* m, double minPartDensity, if (factor == 1) { callback.run(0); } else { - warnAboutShrinking(factor); + warnAboutShrinking(factor, m->getPCU()); Parma_ShrinkPartition(m, factor, callback); } } @@ -159,7 +158,7 @@ void tetrahedronize(Input& in, apf::Mesh2* m) ma::Input* ma_in = ma::makeAdvanced(ma::configureIdentity(m)); ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, m->getPCU()); ma_in->shouldTurnLayerToTets = true; ma::adapt(ma_in); m->verify(); @@ -215,7 +214,7 @@ namespace chef { ma::Input* ma_in = ma::makeAdvanced(ma::configureMatching(m, in.recursiveUR)); ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, m->getPCU()); // get the level set apf::Field* soln = m->findField("solution"); PCU_ALWAYS_ASSERT(soln); @@ -247,7 +246,7 @@ namespace chef { ma::Input* ma_in = ma::makeAdvanced(ma::configureMatching(m, in.recursiveUR)); ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, m->getPCU()); ma_in->shouldRefineLayer = true; ma_in->splitAllLayerEdges = in.splitAllLayerEdges; if (in.snap) { diff --git a/phasta/phAxisymmetry.cc b/phasta/phAxisymmetry.cc index d2834c2d5..132baa001 100644 --- a/phasta/phAxisymmetry.cc +++ b/phasta/phAxisymmetry.cc @@ -1,6 +1,5 @@ #include "phAxisymmetry.h" #include -#include #include #include @@ -126,18 +125,18 @@ apf::MeshTag* tagAngles(apf::Mesh* m, BCs& bcs, apf::MatchedSharing* ms) mdim = m->getModelType(me); mtag = m->getModelTag(me); APF_ITERATE(apf::Matches, matches, mit) { - PCU_COMM_PACK(mit->peer, mit->entity); - PCU_COMM_PACK(mit->peer, mdim); - PCU_COMM_PACK(mit->peer, mtag); + m->getPCU()->Pack(mit->peer, mit->entity); + m->getPCU()->Pack(mit->peer, mdim); + m->getPCU()->Pack(mit->peer, mtag); } } m->end(it); m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(v); + m->getPCU()->Unpack(v); int mdim, mtag; - PCU_COMM_UNPACK(mdim); - PCU_COMM_UNPACK(mtag); + m->getPCU()->Unpack(mdim); + m->getPCU()->Unpack(mtag); gmi_ent* oge = gmi_find(gm, mdim, mtag); gmi_ent* ge = (gmi_ent*) m->toModel(v); double angle; diff --git a/phasta/phBC.cc b/phasta/phBC.cc index 844fba905..00c39d2d3 100644 --- a/phasta/phBC.cc +++ b/phasta/phBC.cc @@ -1,4 +1,4 @@ -#include +//#include #include #include "phBC.h" #ifdef HAVE_SIMMETRIX @@ -123,9 +123,9 @@ void readBCs(gmi_model* m, const char* attFile, bool axisymmetry, BCs& bcs) attachAllAngleBCs(m, bcs); } -void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs) +void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs, pcu::PCU *PCUObj) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); const char* modelfile = in.modelFileName.c_str(); const char* attribfile = in.attributeFileName.c_str(); /* loading the model */ @@ -148,8 +148,8 @@ void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs) #endif /* load attributes */ readBCs(m, attribfile, in.axisymmetry, bcs); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!PCUObj->Self()) lion_oprint(1,"\"%s\" and \"%s\" loaded in %f seconds\n", modelfile, attribfile, t1 - t0); } diff --git a/phasta/phBC.h b/phasta/phBC.h index 36e97c2d6..761de8d1c 100644 --- a/phasta/phBC.h +++ b/phasta/phBC.h @@ -4,11 +4,10 @@ #include #include #include - #include #include #include "phInput.h" - +//#include /* full names and abbreviations for boundary conditions: Essential boundary conditions: @@ -128,7 +127,7 @@ struct BCs }; void readBCs(gmi_model* m, const char* attFile, bool axisymmetry, BCs& bcs); -void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs); +void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs, pcu::PCU *PCUObj); bool applyNaturalBCs(gmi_model* gm, gmi_ent* ge, BCs& appliedBCs, apf::Vector3 const& x, double* values, int* bits); diff --git a/phasta/phBlock.cc b/phasta/phBlock.cc index 6c5ffe03f..277472d77 100644 --- a/phasta/phBlock.cc +++ b/phasta/phBlock.cc @@ -2,7 +2,6 @@ #include "phBC.h" #include #include -#include #include namespace ph { diff --git a/phasta/phBubble.cc b/phasta/phBubble.cc index 70e00deba..4aa110630 100644 --- a/phasta/phBubble.cc +++ b/phasta/phBubble.cc @@ -1,4 +1,3 @@ -#include #include #include "phBubble.h" #include "phInput.h" @@ -17,14 +16,14 @@ struct Bubble { typedef std::vector Bubbles; -void readBubbles(Bubbles& bubbles, std::string bubbleFileName) +void readBubbles(Bubbles& bubbles, std::string bubbleFileName, pcu::PCU *PCUObj) { char bubblefname[1024]; FILE *filebubble; Bubble readbubble; sprintf(bubblefname, "%s", bubbleFileName.c_str()); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_oprint(1,"reading bubbles info from %s\n",bubblefname); filebubble = fopen(bubblefname, "r"); @@ -38,11 +37,11 @@ void readBubbles(Bubbles& bubbles, std::string bubbleFileName) { bubbles.push_back(readbubble); } - if(!feof(filebubble) && !PCU_Comm_Self()) // If while loop was exited for another reason then eof + if(!feof(filebubble) && !PCUObj->Self()) // If while loop was exited for another reason then eof lion_oprint(1,"WARNING: data in %s does not match expected format\n",bubblefname); fclose(filebubble); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_oprint(1,"%lu bubbles found in %s\n", bubbles.size(), bubblefname); } @@ -90,7 +89,7 @@ void setBubbleScalars(apf::Mesh* m, apf::MeshEntity* v, void initBubbles(apf::Mesh* m, Input& in) { Bubbles bubbles; - readBubbles(bubbles, in.bubbleFileName); + readBubbles(bubbles, in.bubbleFileName, m->getPCU()); PCU_ALWAYS_ASSERT(in.ensa_dof >= 7); apf::NewArray s(in.ensa_dof); apf::Field* f = m->findField("solution"); diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 325cacafd..7dfbfdb8b 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -22,7 +22,7 @@ #include #include #include -#include +//#include #include #include #include @@ -31,15 +31,15 @@ #include #include -static void print_stats(const char* name, double value) +static void print_stats(const char* name, double value, pcu::PCU *pcu_obj) { double min, max, avg; - min = PCU_Min_Double(value); - max = PCU_Max_Double(value); - avg = PCU_Add_Double(value); - avg /= PCU_Comm_Peers(); + min = pcu_obj->Min(value); + max = pcu_obj->Max(value); + avg = pcu_obj->Add(value); + avg /= pcu_obj->Peers(); double imb = max / avg; - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb); } @@ -47,47 +47,39 @@ static void print_stats(const char* name, double value) namespace { -void switchToMasters(int splitFactor) +pcu::PCU* createGroupComm(int splitFactor, pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / splitFactor; int group = self % splitFactor; MPI_Comm groupComm; - MPI_Comm_split(PCU_Get_Comm(), group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + MPI_Comm_split(PCUObj->GetMPIComm(), group, groupRank, &groupComm); + return new pcu::PCU(groupComm); } -void switchToAll(MPI_Comm orig) +void loadCommon(ph::Input& in, ph::BCs& bcs, gmi_model*& g, pcu::PCU *PCUObj) { - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(orig); - MPI_Comm_free(&prevComm); - PCU_Barrier(); + ph::loadModelAndBCs(in, g, bcs, PCUObj); } -void loadCommon(ph::Input& in, ph::BCs& bcs, gmi_model*& g) -{ - ph::loadModelAndBCs(in, g, bcs); -} - -static apf::Mesh2* loadMesh(gmi_model*& g, ph::Input& in) { +static apf::Mesh2* loadMesh(gmi_model*& g, ph::Input& in, pcu::PCU *PCUObj) { const char* meshfile = in.meshFileName.c_str(); if (ph::mesh_has_ext(meshfile, "sms")) { if (in.simmetrixMesh == 0) { - if (PCU_Comm_Self()==0) + if (PCUObj->Self()==0) lion_eprint(1, "oops, turn on flag: simmetrixMesh\n"); in.simmetrixMesh = 1; in.filterMatches = 0; //not support } } - return ph::loadMesh(g, meshfile); + return ph::loadMesh(g, meshfile, PCUObj); } void originalMain(apf::Mesh2*& m, ph::Input& in, - gmi_model* g, apf::Migration*& plan) + gmi_model* g, apf::Migration*& plan, pcu::PCU *pcu_obj) { if(!m) - m = loadMesh(g, in); + m = loadMesh(g, in, pcu_obj); else apf::printStats(m); // Need to set a flag to enable avoiding this when short on time m->verify(); @@ -99,7 +91,7 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, ph::attachZeroSolution(in, m); if (in.buildMapping) ph::buildMapping(m); - apf::setMigrationLimit(SIZET(in.elementsPerMigration), m->getPCU()); + apf::setMigrationLimit(SIZET(in.elementsPerMigration), pcu_obj); if (in.adaptFlag) ph::adapt(in, m); if (in.tetrahedronize) @@ -111,7 +103,7 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, }//end namespace namespace chef { - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU *PCUObj) { FILE* f = NULL; PHASTAIO_OPENTIME(f = pcu_group_open(path, false);) return f; @@ -125,16 +117,16 @@ namespace chef { static FILE* openstream_write(ph::Output& out, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = openGRStreamWrite(out.grs, path);) + PHASTAIO_OPENTIME(f = openGRStreamWrite(out.grs, path, out.mesh->getPCU());) return f; } - static FILE* openstream_read(ph::Input& in, const char* path) { + static FILE* openstream_read(ph::Input& in, const char* path, pcu::PCU *PCUObj) { std::string fname(path); std::string restartStr("restart"); FILE* f = NULL; if( fname.find(restartStr) != std::string::npos ) - PHASTAIO_OPENTIME(f = openRStreamRead(in.rs);) + PHASTAIO_OPENTIME(f = openRStreamRead(in.rs, PCUObj);) else { lion_eprint(1, "ERROR %s type of stream %s is unknown... exiting\n", @@ -164,16 +156,16 @@ namespace ph { { apf::MeshTag* order = NULL; - print_stats("malloc used before Bfs", pcu::GetMem()); + print_stats("malloc used before Bfs", pcu::GetMem(), m->getPCU()); if (in.isReorder && m->getPCU()->Peers() > 1) order = Parma_BfsReorder(m); - print_stats("malloc used before reorder", pcu::GetMem()); + print_stats("malloc used before reorder", pcu::GetMem(), m->getPCU()); apf::reorderMdsMesh(m,order); - print_stats("malloc used after reorder", pcu::GetMem()); + print_stats("malloc used after reorder", pcu::GetMem(), m->getPCU()); } } @@ -184,16 +176,18 @@ namespace ph { } void preprocess(apf::Mesh2* m, Input& in, Output& out, BCs& bcs) { - phastaio_initStats(); + PCUHandle h; + h.ptr = static_cast(m->getPCU()); + phastaio_initStats(h); if(m->getPCU()->Peers() > 1) ph::migrateInterfaceItr(m, bcs); if (in.simmetrixMesh == 0) ph::checkReorder(m,in,m->getPCU()->Peers()); if (in.adaptFlag) - ph::goToStepDir(in.timeStepNumber,in.ramdisk); - std::string path = ph::setupOutputDir(in.ramdisk); + ph::goToStepDir(in.timeStepNumber, m->getPCU(), in.ramdisk); + std::string path = ph::setupOutputDir(m->getPCU(), in.ramdisk); std::string subDirPath = path; - ph::setupOutputSubdir(subDirPath,in.ramdisk); + ph::setupOutputSubdir(subDirPath, m->getPCU(), in.ramdisk); ph::enterFilteredMatching(m, in, bcs); ph::generateOutput(in, bcs, m, out); ph::exitFilteredMatching(m); @@ -225,7 +219,7 @@ namespace ph { } ph::writeGeomBC(out, subDirPath); //write geombc if(!m->getPCU()->Self()) - ph::writeAuxiliaryFiles(path, in.timeStepNumber); + ph::writeAuxiliaryFiles(path, in.timeStepNumber, m->getPCU()); m->verify(); #ifdef HAVE_SIMMETRIX gmi_model* g = m->getModel(); @@ -233,7 +227,7 @@ namespace ph { #endif if (in.adaptFlag) ph::goToParentDir(); - if(in.printIOtime) phastaio_printStats(); + if(in.printIOtime) phastaio_printStats(h); } void preprocess(apf::Mesh2* m, Input& in, Output& out) { gmi_model* g = m->getModel(); @@ -266,27 +260,25 @@ namespace { namespace chef { void bake(gmi_model*& g, apf::Mesh2*& m, - ph::Input& in, ph::Output& out) { + ph::Input& in, ph::Output& out, pcu::PCU *expandedPCUObj) { int shrinkFactor=0; if(in.splitFactor < 0) { shrinkFactor=-1*in.splitFactor; in.splitFactor=1; // this is used in to set readers so if shrinking need to read all } - PCU_ALWAYS_ASSERT(m->getPCU()->Peers() % in.splitFactor == 0); + PCU_ALWAYS_ASSERT(expandedPCUObj->Peers() % in.splitFactor == 0); apf::Migration* plan = 0; ph::BCs bcs; - loadCommon(in, bcs, g); - const int worldRank = m->getPCU()->Self(); - MPI_Comm comm = PCU_Get_Comm(); - switchToMasters(in.splitFactor); + loadCommon(in, bcs, g, expandedPCUObj); + const int worldRank = expandedPCUObj->Self(); + pcu::PCU *groupPCUObj = createGroupComm(in.splitFactor, expandedPCUObj); if ((worldRank % in.splitFactor) == 0) - originalMain(m, in, g, plan); - switchToAll(comm); + originalMain(m, in, g, plan, groupPCUObj); if (in.simmetrixMesh == 0) - m = repeatMdsMesh(m, g, plan, in.splitFactor); + m = repeatMdsMesh(m, g, plan, in.splitFactor, expandedPCUObj); if (in.simmetrixMesh == 0 && shrinkFactor > 1){ GroupCode code; - apf::Unmodulo outMap(m->getPCU()->Self(), m->getPCU()->Peers()); + apf::Unmodulo outMap(expandedPCUObj->Self(), expandedPCUObj->Peers()); code.mesh=m; code.input=∈ code.boundary=&bcs; @@ -296,48 +288,48 @@ namespace chef { ph::preprocess(m,in,out,bcs); } } - void cook(gmi_model*& g, apf::Mesh2*& m) { + void cook(gmi_model*& g, apf::Mesh2*& m, pcu::PCU *pcu_obj) { ph::Input in; - in.load("adapt.inp"); + in.load("adapt.inp", pcu_obj); in.openfile_read = openfile_read; ph::Output out; out.openfile_write = openfile_write; - bake(g,m,in,out); + bake(g,m,in,out,pcu_obj); if ((in.writeVTK) == 1) apf::writeVtkFiles("rendered",m); } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl) { + ph::Input& ctrl, pcu::PCU *pcu_obj) { ctrl.openfile_read = openfile_read; ph::Output out; out.openfile_write = openfile_write; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, GRStream* grs) { + ph::Input& ctrl, GRStream* grs, pcu::PCU *pcu_obj) { ctrl.openfile_read = openfile_read; ph::Output out; out.openfile_write = openstream_write; out.grs = grs; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* rs) { + ph::Input& ctrl, RStream* rs, pcu::PCU *pcu_obj) { ctrl.openfile_read = openstream_read; ctrl.rs = rs; ph::Output out; out.openfile_write = openfile_write; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); return; } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* rs, GRStream* grs) { + ph::Input& ctrl, RStream* rs, GRStream* grs, pcu::PCU *pcu_obj) { ctrl.openfile_read = openstream_read; ctrl.rs = rs; ph::Output out; out.openfile_write = openstream_write; out.grs = grs; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); return; } diff --git a/phasta/phFilterMatching.cc b/phasta/phFilterMatching.cc index dedaeadfa..a81bb7093 100644 --- a/phasta/phFilterMatching.cc +++ b/phasta/phFilterMatching.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -238,21 +237,21 @@ void filterMatching(apf::Mesh2* m, ModelMatching& mm, int dim) gd = gmi_dim(gm, ge); gt = gmi_tag(gm, ge); APF_ITERATE(apf::Matches, matches, mit) { - PCU_COMM_PACK(mit->peer, mit->entity); - PCU_COMM_PACK(mit->peer, e); - PCU_COMM_PACK(mit->peer, gd); - PCU_COMM_PACK(mit->peer, gt); + m->getPCU()->Pack(mit->peer, mit->entity); + m->getPCU()->Pack(mit->peer, e); + m->getPCU()->Pack(mit->peer, gd); + m->getPCU()->Pack(mit->peer, gt); } m->clearMatches(e); } m->end(it); m->getPCU()->Send(); while (m->getPCU()->Receive()) { - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); apf::MeshEntity* oe; - PCU_COMM_UNPACK(oe); - PCU_COMM_UNPACK(gd); - PCU_COMM_UNPACK(gt); + m->getPCU()->Unpack(oe); + m->getPCU()->Unpack(gd); + m->getPCU()->Unpack(gt); gmi_ent* ge = (gmi_ent*) m->toModel(e); if (!mm.count(ge)) continue; diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc index 4f88d1f00..8b5aaa342 100644 --- a/phasta/phGeomBC.cc +++ b/phasta/phGeomBC.cc @@ -1,4 +1,3 @@ -#include #include "phOutput.h" #include "phIO.h" #include "phiotimer.h" @@ -9,10 +8,10 @@ namespace ph { -static std::string buildGeomBCFileName(std::string timestep_or_dat) +static std::string buildGeomBCFileName(std::string timestep_or_dat, pcu::PCU *pcu_obj) { std::stringstream ss; - int rank = PCU_Comm_Self() + 1; + int rank = pcu_obj->Self() + 1; ss << "geombc." << timestep_or_dat << "." << rank; return ss.str(); } @@ -304,7 +303,7 @@ void writeGeomBC(Output& o, std::string path, int timestep) tss << timestep; timestep_or_dat = tss.str(); } - path += buildGeomBCFileName(timestep_or_dat); + path += buildGeomBCFileName(timestep_or_dat, m->getPCU()); phastaio_setfile(GEOMBC_WRITE); FILE* f = o.openfile_write(o, path.c_str()); if (!f) { diff --git a/phasta/phGrowthCurves.cc b/phasta/phGrowthCurves.cc index c3cc0c343..ef1e3248f 100644 --- a/phasta/phGrowthCurves.cc +++ b/phasta/phGrowthCurves.cc @@ -1,4 +1,3 @@ -#include #include #include #include "phOutput.h" diff --git a/phasta/phGrowthCurves_empty.cc b/phasta/phGrowthCurves_empty.cc index 5cfafa07f..297bd5649 100644 --- a/phasta/phGrowthCurves_empty.cc +++ b/phasta/phGrowthCurves_empty.cc @@ -1,4 +1,3 @@ -#include #include #include "phGrowthCurves.h" #include "phOutput.h" diff --git a/phasta/phIO.c b/phasta/phIO.c index 2806dd783..d0517b6bc 100644 --- a/phasta/phIO.c +++ b/phasta/phIO.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -70,7 +70,7 @@ static void parse_header(char* header, char** name, long* bytes, } } -static int find_header(FILE* f, const char* name, char* found, char header[PH_LINE]) +static int find_header(PCUHandle h, FILE* f, const char* name, char* found, char header[PH_LINE]) { char* hname; long bytes; @@ -88,7 +88,7 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI } fseek(f, bytes, SEEK_CUR); } - if (!PCU_Comm_Self() && strlen(name) > 0) + if (!PCU_Comm_Self2(h) && strlen(name) > 0) lion_eprint(1,"warning: phIO could not find \"%s\"\n",name); return 0; } @@ -102,11 +102,11 @@ static void write_magic_number(FILE* f) fprintf(f,"\n"); } -static int seek_after_header(FILE* f, const char* name) +static int seek_after_header(PCUHandle h, FILE* f, const char* name) { char dummy[PH_LINE]; char found[PH_LINE]; - return find_header(f, name, found, dummy); + return find_header(h, f, name, found, dummy); } static void my_fread(void* p, size_t size, size_t nmemb, FILE* f) @@ -115,11 +115,11 @@ static void my_fread(void* p, size_t size, size_t nmemb, FILE* f) PCU_ALWAYS_ASSERT(r == nmemb); } -static int read_magic_number(FILE* f) +static int read_magic_number(PCUHandle h, FILE* f) { int magic; - if (!seek_after_header(f, magic_name)) { - if (!PCU_Comm_Self()) + if (!seek_after_header(h, f, magic_name)) { + if (!PCU_Comm_Self2(h)) lion_eprint(1,"warning: not swapping bytes\n"); rewind(f); return 0; @@ -163,7 +163,8 @@ static void parse_params(char* header, long* bytes, } int ph_should_swap(FILE* f) { - return read_magic_number(f); + PCUHandle h = PCU_Get_Global_Handle(); + return read_magic_number(h, f); } int ph_read_field(FILE* f, const char* field, int swap, @@ -172,7 +173,8 @@ int ph_read_field(FILE* f, const char* field, int swap, long bytes, n; char header[PH_LINE]; int ok; - ok = find_header(f, field, hname, header); + PCUHandle h = PCU_Get_Global_Handle(); + ok = find_header(h, f, field, hname, header); if(!ok) /* not found */ return 0; parse_params(header, &bytes, nodes, vars, step); diff --git a/phasta/phInput.cc b/phasta/phInput.cc index ffc9989c2..f34f0d9af 100644 --- a/phasta/phInput.cc +++ b/phasta/phInput.cc @@ -1,4 +1,3 @@ -#include #include "phInput.h" #include #include @@ -210,10 +209,10 @@ static void makeDeprecated(stringset& old) old.insert("writePhastaFiles"); } -static bool deprecated(stringset& old, std::string const& name) +static bool deprecated(stringset& old, std::string const& name, pcu::PCU *PCUObj) { if( old.count(name) ) { - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) lion_eprint(1, "WARNING deprecated input \"%s\" ... " "carefully check stderr and stdout for unexpected behavior\n", name.c_str()); @@ -227,7 +226,8 @@ static void readInputFile( const char* filename, StringMap& stringMap, IntMap& intMap, - DblMap& dblMap) + DblMap& dblMap, + pcu::PCU *PCUObj) { stringset old; makeDeprecated(old); @@ -236,7 +236,7 @@ static void readInputFile( fail("could not open \"%s\"", filename); std::string name; while (f >> name) { - if (name[0] == '#' || deprecated(old,name)) { + if (name[0] == '#' || deprecated(old,name,PCUObj)) { std::getline(f, name, '\n'); continue; } @@ -257,14 +257,14 @@ static void validate(Input& in) PCU_ALWAYS_ASSERT( ! (in.buildMapping && in.adaptFlag)); } -void Input::load(const char* filename) +void Input::load(const char* filename, pcu::PCU *PCUObj) { setDefaults(*this); StringMap stringMap; IntMap intMap; DblMap dblMap; formMaps(*this, stringMap, intMap, dblMap); - readInputFile(filename, stringMap, intMap, dblMap); + readInputFile(filename, stringMap, intMap, dblMap, PCUObj); validate(*this); } diff --git a/phasta/phInput.h b/phasta/phInput.h index bca7c842e..9b823835a 100644 --- a/phasta/phInput.h +++ b/phasta/phInput.h @@ -13,6 +13,10 @@ #include #include +namespace pcu{ + class PCU; +} + struct RStream; namespace ph { @@ -22,7 +26,7 @@ class Input { public: Input(); - void load(const char* filename); + void load(const char* filename, pcu::PCU *PCUObj); int txtCoord; //HACK added to get through compile int timeStepNumber; /** \brief this corresponds to the number of degrees of @@ -158,7 +162,7 @@ class Input double meshqCrtn; double elementImbalance; double vertexImbalance; - FILE* (*openfile_read)(Input& in, const char* path); + FILE* (*openfile_read)(Input& in, const char* path, pcu::PCU *PCUObj); RStream* rs; /** \brief the flag for switch between simmetrix mesh and pumi-based mesh. avoid run incompatible APIs with simmetrix mesh */ diff --git a/phasta/phInterfaceCutter.cc b/phasta/phInterfaceCutter.cc index 55547b963..1b51cc809 100644 --- a/phasta/phInterfaceCutter.cc +++ b/phasta/phInterfaceCutter.cc @@ -1,6 +1,5 @@ #include "phInterfaceCutter.h" #include -#include #include #include #include diff --git a/phasta/phLinks.cc b/phasta/phLinks.cc index 838018bc1..73a2a0ff3 100644 --- a/phasta/phLinks.cc +++ b/phasta/phLinks.cc @@ -1,4 +1,3 @@ -#include #include "phLinks.h" #include "phAdjacent.h" #include diff --git a/phasta/phMeshQuality.cc b/phasta/phMeshQuality.cc index f23afa217..0b2a6d259 100644 --- a/phasta/phMeshQuality.cc +++ b/phasta/phMeshQuality.cc @@ -20,7 +20,6 @@ #endif #endif -#include #include #include #include diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc index e4606acec..35b09fb96 100644 --- a/phasta/phOutput.cc +++ b/phasta/phOutput.cc @@ -1,4 +1,3 @@ -#include #include #include "phOutput.h" #include "phGrowthCurves.h" @@ -627,12 +626,12 @@ static void getMaxElementNodes(Output& o) /* returns the global periodic master iff it is on this part, otherwise returns e */ static apf::MeshEntity* getLocalPeriodicMaster(apf::MatchedSharing* sh, - apf::MeshEntity* e) + apf::MeshEntity* e, pcu::PCU *PCUObj) { if ( ! sh) return e; apf::Copy globalMaster = sh->getOwnerCopy(e); - if (globalMaster.peer == PCU_Comm_Self()) + if (globalMaster.peer == PCUObj->Self()) return globalMaster.entity; else return e; @@ -649,7 +648,7 @@ static void getLocalPeriodicMasters(Output& o, apf::Numbering* n, BCs& bcs) while ((e = m->iterate(it))) { apf::ModelEntity* me = m->toModel(e); bool isDG = ph::isInterface(m->getModel(),(gmi_ent*) me,bcs.fields["DG interface"]); - apf::MeshEntity* master = getLocalPeriodicMaster(sh, e); + apf::MeshEntity* master = getLocalPeriodicMaster(sh, e, m->getPCU()); if (master == e || isDG) iper[i] = 0; else diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc index 7cd11eeec..25a52c436 100644 --- a/phasta/phPartition.cc +++ b/phasta/phPartition.cc @@ -1,4 +1,3 @@ -#include #include "phPartition.h" #include "phInput.h" #include "ph.h" diff --git a/phasta/phRestart.cc b/phasta/phRestart.cc index 3de57dec7..e403df471 100644 --- a/phasta/phRestart.cc +++ b/phasta/phRestart.cc @@ -1,4 +1,3 @@ -#include #include #include "phRestart.h" #include @@ -458,21 +457,23 @@ static double* buildMappingVtxId(apf::Mesh* m) return data; } -static std::string buildRestartFileName(std::string prefix, int step) +static std::string buildRestartFileName(std::string prefix, int step, pcu::PCU *PCUObj) { std::stringstream ss; - int rank = PCU_Comm_Self() + 1; + int rank = PCUObj->Self() + 1; ss << prefix << '.' << step << '.' << rank; return ss.str(); } void readAndAttachFields(Input& in, apf::Mesh* m) { - phastaio_initStats(); + PCUHandle h; + h.ptr = static_cast(m->getPCU()); + phastaio_initStats(h); double t0 = pcu::Time(); - setupInputSubdir(in.restartFileName); - std::string filename = buildRestartFileName(in.restartFileName, in.timeStepNumber); + setupInputSubdir(in.restartFileName, m->getPCU()); + std::string filename = buildRestartFileName(in.restartFileName, in.timeStepNumber, m->getPCU()); phastaio_setfile(RESTART_READ); - FILE* f = in.openfile_read(in, filename.c_str()); + FILE* f = in.openfile_read(in, filename.c_str(), m->getPCU()); if (!f) { lion_eprint(1,"failed to open \"%s\"!\n", filename.c_str()); abort(); @@ -484,7 +485,7 @@ void readAndAttachFields(Input& in, apf::Mesh* m) { double t1 = pcu::Time(); if (!m->getPCU()->Self()) lion_oprint(1,"fields read and attached in %f seconds\n", t1 - t0); - if(in.printIOtime) phastaio_printStats(); + if(in.printIOtime) phastaio_printStats(h); } static void destroyIfExists(apf::Mesh* m, const char* name) @@ -518,7 +519,7 @@ void attachZeroSolution(Input& in, apf::Mesh* m) void detachAndWriteSolution(Input& in, Output& out, apf::Mesh* m, std::string path) { double t0 = pcu::Time(); - path += buildRestartFileName("restart", in.timeStepNumber); + path += buildRestartFileName("restart", in.timeStepNumber, m->getPCU()); phastaio_setfile(RESTART_WRITE); FILE* f = out.openfile_write(out, path.c_str()); if (!f) { diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 3c5d7c4e5..1fca8abaa 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -20,7 +19,7 @@ #include #include #include - +#include #include @@ -29,7 +28,7 @@ static void attachOrder(apf::Mesh* m) apf::numberOverlapDimension(m, "sim_order", m->getDimension()); } namespace { - static FILE* openFileRead(ph::Input&, const char* path) { + static FILE* openFileRead(ph::Input&, const char* path, pcu::PCU*) { return fopen(path, "r"); } @@ -68,7 +67,7 @@ int should_fix_pyramids = 1; int should_attach_order = 0; bool found_bad_arg = false; -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { opterr = 0; @@ -105,18 +104,18 @@ void getConfig(int argc, char** argv) { gmi_native_path = optarg; break; case '?': - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) lion_oprint(1,"warning: skipping unrecognized option\n"); break; default: - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) lion_oprint(1,"Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) lion_oprint(1,"Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -125,7 +124,7 @@ void getConfig(int argc, char** argv) { sms_path = argv[i++]; smb_path = argv[i++]; - if (!PCU_Comm_Self()) { + if (!pcu_obj->Self()) { lion_oprint(1,"fix_pyramids %d attach_order %d enable_log %d\n", should_fix_pyramids, should_attach_order, should_log); lion_oprint(1,"native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", @@ -136,7 +135,7 @@ void getConfig(int argc, char** argv) { static void fixCoords(apf::Mesh2* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* e; apf::Vector3 x; @@ -148,9 +147,9 @@ static void fixCoords(apf::Mesh2* m) m->getPoint(e, 0, x); m->getParam(e, p); APF_ITERATE(apf::Copies, remotes, rit) { - PCU_COMM_PACK(rit->first, rit->second); - PCU_COMM_PACK(rit->first, x); - PCU_COMM_PACK(rit->first, p); + m->getPCU()->Pack(rit->first, rit->second); + m->getPCU()->Pack(rit->first, x); + m->getPCU()->Pack(rit->first, p); } } m->end(it); @@ -163,9 +162,9 @@ static void fixCoords(apf::Mesh2* m) int p_diffs = 0; while (m->getPCU()->Receive()) { apf::Vector3 ox, op; - PCU_COMM_UNPACK(e); - PCU_COMM_UNPACK(ox); - PCU_COMM_UNPACK(op); + m->getPCU()->Unpack(e); + m->getPCU()->Unpack(ox); + m->getPCU()->Unpack(op); m->getPoint(e, 0, x); m->getParam(e, p); if (!(apf::areClose(p, op, 0.0))) { @@ -190,11 +189,11 @@ static void fixCoords(apf::Mesh2* m) double global_max[2]; global_max[0] = max_x_diff; global_max[1] = max_p_diff; - PCU_Max_Doubles(global_max, 2); + m->getPCU()->Max(global_max, 2); long global_diffs[2]; global_diffs[0] = x_diffs; global_diffs[1] = p_diffs; - PCU_Add_Longs(global_diffs, 2); + m->getPCU()->Add(global_diffs, 2); /* admittedly not the best way of checking which processor had the max */ if (global_diffs[0] && (global_max[0] == max_x_diff)) @@ -216,18 +215,19 @@ static void postConvert(apf::Mesh2* m) int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); MS_init(); SimModel_start(); Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv); + getConfig(argc, argv, pcu_obj.get()); if( should_log ) Sim_logOn("convert.sim.log"); if (should_attach_order && should_fix_pyramids) { - if (!PCU_Comm_Self()) + if (!pcu_obj.get()->Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = false; } @@ -243,20 +243,20 @@ int main(int argc, char** argv) else mdl = gmi_load(gmi_path); pGModel simModel = gmi_export_sim(mdl); - double t0 = PCU_Time(); + double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); - double t1 = PCU_Time(); - if(!PCU_Comm_Self()) + double t1 = pcu::Time(); + if(!pcu_obj.get()->Self()) lion_eprint(1, "read and created the simmetrix mesh in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh); - double t2 = PCU_Time(); - if(!PCU_Comm_Self()) + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, pcu_obj.get()); + double t2 = pcu::Time(); + if(!simApfMesh->getPCU()->Self()) lion_eprint(1, "created the apf_sim mesh in %f seconds\n", t2-t1); if (should_attach_order) attachOrder(simApfMesh); ph::buildMapping(simApfMesh); - apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh); - double t3 = PCU_Time(); + apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh, pcu_obj.get()); + double t3 = pcu::Time(); if(!mesh->getPCU()->Self()) lion_eprint(1, "created the apf_mds mesh in %f seconds\n", t3-t2); @@ -290,6 +290,6 @@ int main(int argc, char** argv) MS_exit(); if( should_log ) Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index f1dc76a60..b1648a93d 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/phasta/phstream.cc b/phasta/phstream.cc index 44a346dbd..ce5b044f6 100644 --- a/phasta/phstream.cc +++ b/phasta/phstream.cc @@ -14,17 +14,17 @@ namespace { return MPI_Wtime(); } #if PHSTREAM_TIMERS_ON==1 - inline bool isRankZero() { + inline bool isRankZero(pcu::PCU *pcu_obj) { int rank = 0; - MPI_Comm_rank(PCU_Get_Comm(), &rank); + MPI_Comm_rank(pcu_obj->GetMPIComm(), &rank); return !rank; } #endif - inline void printTime(const char* key, double t) { + inline void printTime(const char* key, double t, pcu::PCU *pcu_obj) { (void) key; (void) t; #if PHSTREAM_TIMERS_ON==1 - if( isRankZero() ) + if( isRankZero(pcu_obj) ) lion_eprint(1, "%s %f seconds\n", key, t); #endif } @@ -44,80 +44,80 @@ extern "C" { }; } -RStream* makeRStream() { +RStream* makeRStream(pcu::PCU *PCUObj) { const double t0 = getTime(); RStream* rs = (RStream*) malloc(sizeof(RStream)); rs->restart = NULL; rs->rSz = 0; - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return rs; } #ifdef __APPLE__ -FILE* openRStreamRead(RStream*) { +FILE* openRStreamRead(RStream*, pcu::PCU*) { return NULL; } #else -FILE* openRStreamRead(RStream* rs) { +FILE* openRStreamRead(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); FILE* f = fmemopen(rs->restart, rs->rSz, "r"); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif #ifdef __APPLE__ -FILE* openRStreamWrite(RStream*) { +FILE* openRStreamWrite(RStream*, pcu::PCU*) { return NULL; } #else -FILE* openRStreamWrite(RStream* rs) { +FILE* openRStreamWrite(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); FILE* f = open_memstream(&(rs->restart), &(rs->rSz)); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif -void clearRStream(RStream* rs) { +void clearRStream(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); if(rs->restart) { free(rs->restart); rs->restart = NULL; rs->rSz = 0; } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -void destroyRStream(RStream* rs) { +void destroyRStream(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); - clearRStream(rs); + clearRStream(rs, PCUObj); free(rs); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -void attachRStream(GRStream* grs, RStream* rs) { +void attachRStream(GRStream* grs, RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); rs->restart = grs->restart; rs->rSz = grs->rSz; grs->restart = NULL; grs->rSz = 0; - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -GRStream* makeGRStream() { +GRStream* makeGRStream(pcu::PCU *PCUObj) { const double t0 = getTime(); GRStream* grs = (GRStream*) malloc(sizeof(GRStream)); grs->geom = NULL; grs->gSz = 0; grs->restart = NULL; grs->rSz = 0; - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return grs; } -void whichStream(const char* name, bool& isR, bool& isG) { +void whichStream(const char* name, bool& isR, bool& isG, pcu::PCU *PCUObj) { const double t0 = getTime(); std::string fname(name); std::string restartStr("restart"); @@ -125,7 +125,7 @@ void whichStream(const char* name, bool& isR, bool& isG) { isR = (fname.find(restartStr) != std::string::npos); isG = (fname.find(geombcStr) != std::string::npos); PCU_ALWAYS_ASSERT(isR != isG); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } void writeUnknown(const char* fname) { @@ -135,14 +135,14 @@ void writeUnknown(const char* fname) { } #ifdef __APPLE__ -FILE* openGRStreamRead(GRStream*, const char*) { +FILE* openGRStreamRead(GRStream*, const char*, pcu::PCU*) { return NULL; } #else -FILE* openGRStreamRead(GRStream* grs, const char* named) { +FILE* openGRStreamRead(GRStream* grs, const char* named, pcu::PCU *PCUObj) { const double t0 = getTime(); bool isR, isG; - whichStream(named, isR, isG); + whichStream(named, isR, isG, PCUObj); FILE* f = NULL; if( isR && !isG ) f = fmemopen(grs->restart, grs->rSz, "r"); @@ -152,20 +152,20 @@ FILE* openGRStreamRead(GRStream* grs, const char* named) { writeUnknown(named); exit(1); } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif #ifdef __APPLE__ -FILE* openGRStreamWrite(GRStream*, const char*) { +FILE* openGRStreamWrite(GRStream*, const char*, pcu::PCU*) { return NULL; } #else -FILE* openGRStreamWrite(GRStream* grs, const char* named) { +FILE* openGRStreamWrite(GRStream* grs, const char* named, pcu::PCU *PCUObj) { const double t0 = getTime(); bool isR, isG; - whichStream(named, isR, isG); + whichStream(named, isR, isG, PCUObj); FILE* f = NULL; if( isR && !isG ) f = open_memstream(&(grs->restart), &(grs->rSz)); @@ -175,12 +175,12 @@ FILE* openGRStreamWrite(GRStream* grs, const char* named) { writeUnknown(named); exit(1); } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif -void clearGRStream(GRStream* grs) { +void clearGRStream(GRStream* grs, pcu::PCU *PCUObj) { const double t0 = getTime(); if(grs->geom) { free(grs->geom); @@ -192,13 +192,13 @@ void clearGRStream(GRStream* grs) { grs->restart = NULL; grs->rSz = 0; } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -void destroyGRStream(GRStream* grs) { +void destroyGRStream(GRStream* grs, pcu::PCU *PCUObj) { const double t0 = getTime(); - clearGRStream(grs); + clearGRStream(grs, PCUObj); free(grs); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } diff --git a/phasta/phstream.h b/phasta/phstream.h index 6a039e89c..e1747f180 100644 --- a/phasta/phstream.h +++ b/phasta/phstream.h @@ -1,6 +1,7 @@ #ifndef PHSTREAM_H_ #define PHSTREAM_H_ #include +#include /** \file phstream.h @@ -13,29 +14,29 @@ typedef struct RStream* rstream; typedef struct GRStream* grstream; /** @brief make restart stream */ -rstream makeRStream(); +rstream makeRStream(pcu::PCU *PCUObj); /** @brief clear restart stream */ -void clearRStream(rstream rs); +void clearRStream(rstream rs, pcu::PCU *PCUObj); /** @brief detach output stream */ -void destroyRStream(rstream rs); +void destroyRStream(rstream rs, pcu::PCU *PCUObj); /** @brief make geom-restart stream */ -grstream makeGRStream(); +grstream makeGRStream(pcu::PCU *PCUObj); /** @brief clear geom-restart stream */ -void clearGRStream(grstream grs); +void clearGRStream(grstream grs, pcu::PCU *PCUObj); /** @brief destroy geom-restart stream */ -void destroyGRStream(grstream grs); +void destroyGRStream(grstream grs, pcu::PCU *PCUObj); /** @brief open restart stream for reading*/ -FILE* openRStreamRead(rstream rs); +FILE* openRStreamRead(rstream rs, pcu::PCU *PCUObj); /** @brief open restart stream for writing*/ -FILE* openRStreamWrite(rstream rs); +FILE* openRStreamWrite(rstream rs, pcu::PCU *PCUObj); /** @brief open named stream in geom-restart stream for reading*/ -FILE* openGRStreamRead(grstream grs, const char* named); +FILE* openGRStreamRead(grstream grs, const char* named, pcu::PCU *PCUObj); /** @brief open named stream in geom-restart stream for writing*/ -FILE* openGRStreamWrite(grstream grs, const char* named); +FILE* openGRStreamWrite(grstream grs, const char* named, pcu::PCU *PCUObj); /** @brief dev function */ -void attachRStream(grstream grs, rstream rs); +void attachRStream(grstream grs, rstream rs, pcu::PCU *PCUObj); #endif diff --git a/phasta/readUrPrep.cc b/phasta/readUrPrep.cc index 01c57ce1d..9e762e5be 100644 --- a/phasta/readUrPrep.cc +++ b/phasta/readUrPrep.cc @@ -2,15 +2,16 @@ #include #include #include -#include +#include #include +#include namespace { void freeMesh(apf::Mesh* m) { m->destroyNative(); apf::destroyMesh(m); } - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { return fopen(path, "r"); } } @@ -18,19 +19,20 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - PCU_Protect(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::Protect(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], pcu_obj.get()); ph::Input ctrl; - ctrl.load(argv[3]); + ctrl.load(argv[3], pcu_obj.get()); ctrl.openfile_read = openfile_read; ctrl.buildMapping = 0; //can't map new vertices from UR chef::readAndAttachFields(ctrl,m); chef::uniformRefinement(ctrl,m); chef::preprocess(m,ctrl); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/threshold.cc b/phasta/threshold.cc index 7da7ee2c4..905a13cea 100644 --- a/phasta/threshold.cc +++ b/phasta/threshold.cc @@ -1,10 +1,10 @@ -#include #include #include #include #include #include #include +#include #ifdef HAVE_SIMMETRIX #include #include @@ -20,7 +20,8 @@ static double process_element(apf::Vector3 x[4], double sol[4][9]) int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -29,7 +30,7 @@ int main(int argc, char** argv) gmi_register_mesh(); lion_set_verbosity(1); ph::Input in; - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], expanded_pcu_obj.get()); m->verify(); in.restartFileName = argv[3]; in.timeStepNumber = 0; @@ -58,6 +59,6 @@ int main(int argc, char** argv) gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } From 9005de8a84ce4e9f0b39308c00772b7be6138640 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Sun, 24 Mar 2024 21:42:11 -0400 Subject: [PATCH 075/141] small touchups in test files, apfMDS.cc, and apf_sim --- apf_sim/apfSIM.cc | 6 +++--- apf_sim/apfSIM.h | 2 +- mds/apfMDS.cc | 6 +++++- test/constructThenGhost.cc | 14 ++++++++------ test/ph_adapt.cc | 14 ++++++++------ 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/apf_sim/apfSIM.cc b/apf_sim/apfSIM.cc index 8132edfe7..2d2ead446 100644 --- a/apf_sim/apfSIM.cc +++ b/apf_sim/apfSIM.cc @@ -1092,7 +1092,7 @@ static bool findMatches(Mesh* m) return found; } -Mesh2* createMesh(pParMesh mesh) +Mesh2* createMesh(pParMesh mesh, pcu::PCU PCUObj) { /* require one part per process currently for SIM */ PCU_ALWAYS_ASSERT(PM_numParts(mesh)==1); @@ -1105,9 +1105,9 @@ Mesh2* createMesh(pParMesh mesh) serendipity = true; } if (serendipity) - m->init(getSerendipity()); + m->init(getSerendipity(), PCUObj); else - m->init(getLagrange(order)); + m->init(getLagrange(order), PCUObj); m->hasMatches = findMatches(m); return m; } diff --git a/apf_sim/apfSIM.h b/apf_sim/apfSIM.h index 3d87bd4eb..2a0c54969 100644 --- a/apf_sim/apfSIM.h +++ b/apf_sim/apfSIM.h @@ -16,7 +16,7 @@ namespace apf { * * \details This object should be destroyed by apf::destroyMesh. */ -Mesh2* createMesh(pParMesh mesh); +Mesh2* createMesh(pParMesh mesh, pcu::PCU *PCUObj); /** * \brief Casts a Simmetrix entity to an apf::MeshEntity. diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 7ebfaa27e..8fd0b7327 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -184,7 +184,11 @@ class MeshMDS : public Mesh2 MeshMDS(gmi_model* m, Mesh* from, apf::MeshEntity** nodes, apf::MeshEntity** elems, bool copy_data=true) { - init(apf::getLagrange(1)); + if(from->getPCU() != nullptr){ + init(apf::getLagrange(1), from->getPCU()); + } else { + init(apf::getLagrange(1)); + } mds_id cap[MDS_TYPES]; cap[MDS_VERTEX] = from->count(0); cap[MDS_EDGE] = from->count(1); diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index 34fae29fb..dad66cb7e 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -9,12 +9,14 @@ #include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -24,7 +26,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], pcu_obj.get()); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -32,7 +34,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false, PCUObj); + m = apf::makeEmptyMdsMesh(model, dim, false, pcu_obj.get()); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; @@ -43,10 +45,10 @@ int main(int argc, char** argv) outMap.clear(); m->verify(); - if (!pumi_rank(PCUObj)) printf("model/mesh converted to pumi instance\n"); + if (!pumi_rank(pcu_obj.get())) printf("model/mesh converted to pumi instance\n"); //create the pumi instance to use pumi api's - pGeom g = pumi_geom_load(model, PCUObj); + pGeom g = pumi_geom_load(model, pcu_obj.get()); pMesh pm = pumi_mesh_load(m); pumi_mesh_verify(pm); @@ -91,6 +93,6 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(pm); - delete PCUObj; + } MPI_Finalize(); } diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index dc490efce..b24a1b4e0 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -18,6 +18,7 @@ #include #endif #include +#include static bool overwriteAPFCoord(apf::Mesh2* m) { apf::Field* f = m->findField("motion_coords"); @@ -37,7 +38,7 @@ static bool overwriteAPFCoord(apf::Mesh2* m) { return true; } -static FILE* openfile_read(ph::Input&, const char* path) { +static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { return pcu_group_open(path, false); } @@ -47,7 +48,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -58,10 +60,10 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); /* load model, mesh and configure input */ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); ph::Input in; - in.load("adapt.inp"); + in.load("adapt.inp", PCUObj.get()); in.openfile_read = openfile_read; /* attach solution and other fields */ ph::readAndAttachFields(in,m); @@ -100,7 +102,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } From 6bb2806d9662db3fc3ef897b35fdbfc7c3e93dff Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 25 Mar 2024 12:48:00 -0400 Subject: [PATCH 076/141] Parma folder fixes, only thing not changed is parma_group.cc, all tests pass. Making a new branch from here testing two different methods of fixing parma_group --- parma/diffMC/parma_balancer.cc | 7 +++-- parma/diffMC/parma_commons.cc | 7 +++-- parma/diffMC/parma_commons.h | 3 ++- parma/diffMC/parma_dcpart.cc | 2 +- parma/diffMC/parma_dcpartFixer.cc | 11 ++++---- parma/diffMC/parma_elmBalancer.cc | 4 +-- parma/diffMC/parma_elmLtVtxEdgeBalancer.cc | 5 ++-- parma/diffMC/parma_entWeights.cc | 13 +++++---- parma/diffMC/parma_ghost.cc | 18 ++++++------- parma/diffMC/parma_ghostElement.cc | 10 +++---- parma/diffMC/parma_ghostMPAS.cc | 4 +-- parma/diffMC/parma_ghostMPASWeights.cc | 31 +++++++++++----------- parma/diffMC/parma_ghostWeights.cc | 31 +++++++++++----------- parma/diffMC/parma_graphDist.cc | 2 +- parma/diffMC/parma_shapeOptimizer.cc | 17 ++++++------ parma/diffMC/parma_shapeTargets.cc | 21 +++++++-------- parma/diffMC/parma_sides.cc | 7 +++-- parma/diffMC/parma_sides.h | 2 +- parma/diffMC/parma_step.cc | 4 +-- parma/diffMC/parma_stop.cc | 5 ++-- parma/diffMC/parma_stop.h | 7 ++--- parma/diffMC/parma_targets.h | 2 +- parma/diffMC/parma_vtxBalancer.cc | 4 +-- parma/diffMC/parma_vtxEdgeElmBalancer.cc | 4 +-- parma/diffMC/parma_vtxElmBalancer.cc | 4 +-- parma/diffMC/parma_vtxPtnWriter.cc | 12 ++++----- parma/diffMC/parma_weights.h | 4 +-- parma/parma.cc | 29 ++++++++++---------- 28 files changed, 130 insertions(+), 140 deletions(-) diff --git a/parma/diffMC/parma_balancer.cc b/parma/diffMC/parma_balancer.cc index 3a3eb9fdd..14e3a735f 100644 --- a/parma/diffMC/parma_balancer.cc +++ b/parma/diffMC/parma_balancer.cc @@ -1,12 +1,11 @@ -#include #include "parma_balancer.h" #include "parma_monitor.h" #include "parma_graphDist.h" #include "parma_commons.h" namespace { - void printTiming(const char* type, int steps, double tol, double time) { - if (!PCU_Comm_Self()) + void printTiming(const char* type, int steps, double tol, double time, pcu::PCU *PCUObj) { + if (!PCUObj->Self()) parmaCommons::status("%s balanced in %d steps to %f in %f seconds\n", type, steps, tol, time); } @@ -37,7 +36,7 @@ namespace parma { int step = 0; double t0 = pcu::Time(); while (runStep(wtag,tolerance) && step++ < maxStep); - printTiming(name, step, tolerance, pcu::Time()-t0); + printTiming(name, step, tolerance, pcu::Time()-t0, mesh->getPCU()); } void Balancer::monitorUpdate(double v, Slope* s, Average* a) { s->push(v); diff --git a/parma/diffMC/parma_commons.cc b/parma/diffMC/parma_commons.cc index afaa72812..cfd46df55 100644 --- a/parma/diffMC/parma_commons.cc +++ b/parma/diffMC/parma_commons.cc @@ -1,5 +1,4 @@ #include "parma_commons.h" -#include "PCU.h" #include #include #include @@ -45,9 +44,9 @@ int parmaCommons::isMore(double a, double b) { return 0; } -void parmaCommons::printElapsedTime(const char* fn, double elapsed) { - elapsed = PCU_Max_Double(elapsed); - if( !PCU_Comm_Self() ) +void parmaCommons::printElapsedTime(const char* fn, double elapsed, pcu::PCU *PCUObj) { + elapsed = PCUObj->Max(elapsed); + if( !PCUObj->Self() ) status("%s elapsed time %lf seconds\n", fn, elapsed); } diff --git a/parma/diffMC/parma_commons.h b/parma/diffMC/parma_commons.h index 48c2ce755..6e4e604ba 100644 --- a/parma/diffMC/parma_commons.h +++ b/parma/diffMC/parma_commons.h @@ -2,6 +2,7 @@ #define PARMA_COMMONS_H_ #include "apfArray.h" +#include "PCUObj.h" namespace parmaCommons { @@ -16,7 +17,7 @@ template int isEqlArr(type1 a, type2 b) { int isLess(double a, double b); int isMore(double a, double b); -void printElapsedTime(const char* fn, double elapsed); +void printElapsedTime(const char* fn, double elapsed, pcu::PCU *PCUObj); void debug(bool isActive, const char* fmt,...); void status(const char* fmt,...); diff --git a/parma/diffMC/parma_dcpart.cc b/parma/diffMC/parma_dcpart.cc index 789102fad..e43844582 100644 --- a/parma/diffMC/parma_dcpart.cc +++ b/parma/diffMC/parma_dcpart.cc @@ -119,7 +119,7 @@ unsigned dcPart::numDisconnectedComps() { count += sz; } if( verbose ) - parmaCommons::printElapsedTime(__func__, pcu::Time() - t1); + parmaCommons::printElapsedTime(__func__, pcu::Time() - t1, m->getPCU()); PCU_ALWAYS_ASSERT(numDc+numIso >= 1); return (numDc+numIso)-1; } diff --git a/parma/diffMC/parma_dcpartFixer.cc b/parma/diffMC/parma_dcpartFixer.cc index 41f26d7a4..d39e6c59e 100644 --- a/parma/diffMC/parma_dcpartFixer.cc +++ b/parma/diffMC/parma_dcpartFixer.cc @@ -1,4 +1,3 @@ -#include "PCU.h" #include "parma_dcpart.h" #include "parma_commons.h" #include "parma_convert.h" @@ -8,11 +7,11 @@ typedef std::map muu; namespace { - bool isInMis(muu& mt) { - unsigned seed = TO_UINT(PCU_Comm_Self()+1); + bool isInMis(muu& mt, pcu::PCU *PCUObj) { + unsigned seed = TO_UINT(PCUObj->Self()+1); mis_init(seed); misLuby::partInfo part; - part.id = PCU_Comm_Self(); + part.id = PCUObj->Self(); std::set targets; APF_ITERATE(muu, mt, mtItr) { int peer = TO_INT(mtItr->second); @@ -81,7 +80,7 @@ class dcPartFixer::PartFixer : public dcPart { dcCompTgts[i] = getCompPeer(i); PCU_ALWAYS_ASSERT( dcCompTgts.size() == getNumComps()-1 ); apf::Migration* plan = new apf::Migration(m); - if ( isInMis(dcCompTgts) ) + if ( isInMis(dcCompTgts, m->getPCU()) ) setupPlan(dcCompTgts, plan); reset(); @@ -92,7 +91,7 @@ class dcPartFixer::PartFixer : public dcPart { "loop %d components %d seconds %.3f %.3f\n", loop, ndc, t3-t2, pcu::Time()-t3); } - parmaCommons::printElapsedTime(__func__, pcu::Time() - t1); + parmaCommons::printElapsedTime(__func__, pcu::Time() - t1, m->getPCU()); } }; diff --git a/parma/diffMC/parma_elmBalancer.cc b/parma/diffMC/parma_elmBalancer.cc index ae477984d..e897a7780 100644 --- a/parma/diffMC/parma_elmBalancer.cc +++ b/parma/diffMC/parma_elmBalancer.cc @@ -17,14 +17,14 @@ namespace { ElmBalancer(apf::Mesh* m, double f, int v) : Balancer(m, f, v, "elements") { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = parma::avgSharedSides(s); + sideTol = parma::avgSharedSides(s, mesh->getPCU()); delete s; } bool runStep(apf::MeshTag* wtag, double tolerance) { const double maxElmImb = Parma_GetWeightedEntImbalance(mesh, wtag, mesh->getDimension()); parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); parma::Weights* w = parma::makeEntWeights(mesh, wtag, s, mesh->getDimension()); parma::Targets* t = parma::makeTargets(s, w, factor); diff --git a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc index 05d15c656..1d05b0eab 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc @@ -29,7 +29,7 @@ namespace { status("maxEdge %.3f\n", maxEdge); } parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); @@ -48,8 +48,7 @@ namespace { delete w[1]; parma::Selector* sel = parma::makeElmLtVtxEdgeSelector(mesh, wtag, maxVtx, maxEdge); - - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); if( !mesh->getPCU()->Self() && verbose ) diff --git a/parma/diffMC/parma_entWeights.cc b/parma/diffMC/parma_entWeights.cc index 0a10201ce..ed1f062b5 100644 --- a/parma/diffMC/parma_entWeights.cc +++ b/parma/diffMC/parma_entWeights.cc @@ -1,5 +1,4 @@ #include -#include #include "parma_entWeights.h" #include "parma_sides.h" @@ -25,17 +24,17 @@ namespace parma { return sum; } - void getImbalance(Weights* w, double& imb, double& avg) { + void getImbalance(Weights* w, double& imb, double& avg, pcu::PCU *PCUObj) { double sum, max; sum = max = w->self(); - sum = PCU_Add_Double(sum); - max = PCU_Max_Double(max); - avg = sum/PCU_Comm_Peers(); + sum = PCUObj->Add(sum); + max = PCUObj->Max(max); + avg = sum/PCUObj->Peers(); imb = max/avg; } - double getMaxWeight(Weights* w) { - return PCU_Max_Double(w->self()); + double getMaxWeight(Weights* w, pcu::PCU *PCUObj) { + return PCUObj->Max(w->self()); } double getEntWeight(apf::Mesh* m, apf::MeshEntity* e, apf::MeshTag* w) { diff --git a/parma/diffMC/parma_ghost.cc b/parma/diffMC/parma_ghost.cc index 62f653453..edd1fe68f 100644 --- a/parma/diffMC/parma_ghost.cc +++ b/parma/diffMC/parma_ghost.cc @@ -20,14 +20,14 @@ namespace { : Balancer(m, f, v, "ghostElms"), layers(l) { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); @@ -38,13 +38,13 @@ namespace { destroyGhostWeights(gw); double vtxImb, vtxAvg; - parma::getImbalance(vtxW, vtxImb, vtxAvg); + parma::getImbalance(vtxW, vtxImb, vtxAvg, mesh->getPCU()); if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f avg %.3f\n", vtxImb, vtxAvg); delete vtxW; double elmImb, elmAvg; - parma::getImbalance(elmW, elmImb, elmAvg); + parma::getImbalance(elmW, elmImb, elmAvg, mesh->getPCU()); monitorUpdate(elmImb, iS, iA); monitorUpdate(avgSides, sS, sA); @@ -70,14 +70,14 @@ namespace { : Balancer(m, f, v, "ghostVtxLtElms"), layers(l), maxElmW(0), stepNum(0) { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); @@ -89,15 +89,15 @@ namespace { destroyGhostWeights(gw); double elmImb, elmAvg; - parma::getImbalance(elmW,elmImb,elmAvg); + parma::getImbalance(elmW,elmImb,elmAvg,mesh->getPCU()); double edgeImb, edgeAvg; - parma::getImbalance(edgeW, edgeImb, edgeAvg); + parma::getImbalance(edgeW, edgeImb, edgeAvg, mesh->getPCU()); if( !mesh->getPCU()->Self() && verbose ) { status("elm imbalance %.3f avg %.3f\n", elmImb, elmAvg); status("edge imbalance %.3f avg %.3f\n", edgeImb, edgeAvg); } if( !stepNum ) //FIXME need to set the imbalance at the beginning for the primary entity - maxElmW = parma::getMaxWeight(elmW); + maxElmW = parma::getMaxWeight(elmW, mesh->getPCU()); delete edgeW; monitorUpdate(elmImb, iS, iA); diff --git a/parma/diffMC/parma_ghostElement.cc b/parma/diffMC/parma_ghostElement.cc index f33dde4bd..1514d413b 100644 --- a/parma/diffMC/parma_ghostElement.cc +++ b/parma/diffMC/parma_ghostElement.cc @@ -20,14 +20,14 @@ namespace { : Balancer(m, f, v, "ghostEdges") { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); @@ -39,8 +39,8 @@ namespace { destroyGhostWeights(gw); double faceImb, faceAvg, elmImb, elmAvg; - parma::getImbalance(faceW, faceImb, faceAvg); - parma::getImbalance(elmW, elmImb, elmAvg); + parma::getImbalance(faceW, faceImb, faceAvg, mesh->getPCU()); + parma::getImbalance(elmW, elmImb, elmAvg, mesh->getPCU()); if( !mesh->getPCU()->Self() && verbose ) { status("face imbalance %.3f avg %.3f\n", faceImb, faceAvg); status("elm imbalance %.3f avg %.3f\n", elmImb, elmAvg); @@ -49,7 +49,7 @@ namespace { delete elmW; double edgeImb, edgeAvg; - parma::getImbalance(edgeW, edgeImb, edgeAvg); + parma::getImbalance(edgeW, edgeImb, edgeAvg, mesh->getPCU()); monitorUpdate(edgeImb, iS, iA); monitorUpdate(avgSides, sS, sA); diff --git a/parma/diffMC/parma_ghostMPAS.cc b/parma/diffMC/parma_ghostMPAS.cc index 2e06d9e65..ab4a6afff 100644 --- a/parma/diffMC/parma_ghostMPAS.cc +++ b/parma/diffMC/parma_ghostMPAS.cc @@ -17,7 +17,7 @@ namespace { : Balancer(m, f, v, "ghosts"), layers(l), bridge(b) { parma::Sides* s = parma::makeElmBdrySides(mesh); - sideTol = static_cast(parma::avgSharedSides(s)); + sideTol = static_cast(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !mesh->getPCU()->Self() && verbose ) lion_oprint(1, "sideTol %d\n", sideTol); @@ -27,7 +27,7 @@ namespace { const double maxElmImb = Parma_GetWeightedEntImbalance(mesh, wtag, mesh->getDimension()); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); if( !mesh->getPCU()->Self() && verbose ) diff --git a/parma/diffMC/parma_ghostMPASWeights.cc b/parma/diffMC/parma_ghostMPASWeights.cc index fd24b9686..78f7f4651 100644 --- a/parma/diffMC/parma_ghostMPASWeights.cc +++ b/parma/diffMC/parma_ghostMPASWeights.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -126,9 +125,9 @@ namespace parma { { GhostElementFinder finder(m, wtag, layers, bridge); findGhostElements(&finder, s); - exchangeGhostElementsFrom(); + exchangeGhostElementsFrom(m->getPCU()); weight += ownedVtxWeight(m, wtag); - exchange(); + exchange(m->getPCU()); } ~GhostMPASWeights() {} double self() { @@ -145,32 +144,32 @@ namespace parma { set(side->first, finder->weight(side->first)); sides->end(); } - void exchangeGhostElementsFrom() { - PCU_Comm_Begin(); + void exchangeGhostElementsFrom(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostMPASWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_COMM_PACK(ghost->first, ghost->second); + PCUObj->Pack(ghost->first, ghost->second); end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + PCUObj->Send(); + while (PCUObj->Listen()) { double ghostsFromPeer = 0; - PCU_COMM_UNPACK(ghostsFromPeer); + PCUObj->Unpack(ghostsFromPeer); weight += ghostsFromPeer; } } - void exchange() { - PCU_Comm_Begin(); + void exchange(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostMPASWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_COMM_PACK(ghost->first, weight); + PCUObj->Pack(ghost->first, weight); end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + PCUObj->Send(); + while (PCUObj->Listen()) { double peerWeight; - PCU_COMM_UNPACK(peerWeight); - int peer = PCU_Comm_Sender(); + PCUObj->Unpack(peerWeight); + int peer = PCUObj->Sender(); set(peer, peerWeight); } } diff --git a/parma/diffMC/parma_ghostWeights.cc b/parma/diffMC/parma_ghostWeights.cc index d37e587b8..ef78d1bcf 100644 --- a/parma/diffMC/parma_ghostWeights.cc +++ b/parma/diffMC/parma_ghostWeights.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -225,8 +224,8 @@ namespace parma { for(int d=dim+1; d<=3; d++) weight[d] = 0; findGhosts(finder, s); - exchangeGhostsFrom(); - exchange(); + exchangeGhostsFrom(m->getPCU()); + exchange(m->getPCU()); m->getPCU()->DebugPrint("totW vtx %f edge %f elm %f\n", weight[0], weight[1], weight[dim]); } @@ -251,33 +250,33 @@ namespace parma { set(side->first, finder->weight(side->first)); sides->end(); } - void exchangeGhostsFrom() { - PCU_Comm_Begin(); + void exchangeGhostsFrom(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_Comm_Pack(ghost->first, ghost->second, 4*sizeof(double)); + PCUObj->Pack(ghost->first, ghost->second, 4*sizeof(double)); end(); - PCU_Comm_Send(); + PCUObj->Send(); double ghostsFromPeer[4]; - while (PCU_Comm_Listen()) { - PCU_Comm_Unpack(ghostsFromPeer, 4*sizeof(double)); + while (PCUObj->Listen()) { + PCUObj->Unpack(ghostsFromPeer, 4*sizeof(double)); for(int i=0; i<4; i++) weight[i] += ghostsFromPeer[i]; } } - void exchange() { - PCU_Comm_Begin(); + void exchange(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_Comm_Pack(ghost->first, weight, 4*sizeof(double)); + PCUObj->Pack(ghost->first, weight, 4*sizeof(double)); end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int peer = PCU_Comm_Sender(); + PCUObj->Send(); + while (PCUObj->Listen()) { + int peer = PCUObj->Sender(); double* peerWeight = get(peer); - PCU_Comm_Unpack(peerWeight, 4*sizeof(double)); + PCUObj->Unpack(peerWeight, 4*sizeof(double)); } } }; diff --git a/parma/diffMC/parma_graphDist.cc b/parma/diffMC/parma_graphDist.cc index 983830c4c..7cd60570e 100644 --- a/parma/diffMC/parma_graphDist.cc +++ b/parma/diffMC/parma_graphDist.cc @@ -402,6 +402,6 @@ apf::MeshTag* Parma_BfsReorder(apf::Mesh* m, int) { PCU_ALWAYS_ASSERT(checkIds == c.getIdChecksum()); PCU_ALWAYS_ASSERT(check == m->getTagChecksum(dist,apf::Mesh::VERTEX)); m->destroyTag(dist); - parmaCommons::printElapsedTime(__func__,pcu::Time()-t0); + parmaCommons::printElapsedTime(__func__,pcu::Time()-t0,m->getPCU()); return order; } diff --git a/parma/diffMC/parma_shapeOptimizer.cc b/parma/diffMC/parma_shapeOptimizer.cc index b7e8a3dc9..690f2c5d7 100644 --- a/parma/diffMC/parma_shapeOptimizer.cc +++ b/parma/diffMC/parma_shapeOptimizer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma.h" @@ -14,20 +13,20 @@ namespace { using parmaCommons::status; - int getMaxNb(parma::Sides* s) { - return PCU_Max_Int(s->size()); + int getMaxNb(parma::Sides* s, pcu::PCU *PCUObj) { + return PCUObj->Max(s->size()); } class ImbOrMaxNeighbor : public parma::Stop { public: ImbOrMaxNeighbor(parma::Average* nbAvg, double maxNbTol, int targets, int v=0) : nb(nbAvg), nbTol(maxNbTol), tgts(targets), verbose(v) {} - bool stop(double imb, double maxImb) { - int maxTgts = PCU_Max_Int(tgts); + bool stop(double imb, double maxImb, pcu::PCU *PCUObj) { + int maxTgts = PCUObj->Max(tgts); const double nbSlope = nb->avg(); - if( !PCU_Comm_Self() && verbose ) + if( !PCUObj->Self() && verbose ) status("max neighbor slope %f tolerance %f\n", nbSlope, nbTol); - if( !PCU_Comm_Self() && verbose && maxTgts == 0 ) + if( !PCUObj->Self() && verbose && maxTgts == 0 ) status("no targets found... stopping\n"); return imb > maxImb || ( fabs(nbSlope) < nbTol ) || ( maxTgts == 0 ); } @@ -49,9 +48,9 @@ namespace { parma::Weights* w = parma::makeEntWeights(mesh, wtag, s, mesh->getDimension()); parma::Targets* t = - parma::makeShapeTargets(s); + parma::makeShapeTargets(s, mesh->getPCU()); parma::Selector* sel = parma::makeShapeSelector(mesh, wtag); - double maxNb = TO_DOUBLE(getMaxNb(s)); + double maxNb = TO_DOUBLE(getMaxNb(s, mesh->getPCU())); monitorUpdate(maxNb, sS, sA); parma::Stop* stopper = new ImbOrMaxNeighbor(sA, maxNb*.001, t->size(), verbose); diff --git a/parma/diffMC/parma_shapeTargets.cc b/parma/diffMC/parma_shapeTargets.cc index be9bcbc81..80d466631 100644 --- a/parma/diffMC/parma_shapeTargets.cc +++ b/parma/diffMC/parma_shapeTargets.cc @@ -1,4 +1,3 @@ -#include #include #include "parma_sides.h" #include "parma_weights.h" @@ -14,9 +13,9 @@ namespace parma { class ShapeTargets : public Targets { public: - ShapeTargets(Sides* s) { + ShapeTargets(Sides* s, pcu::PCU *PCUObj) { smallLimit = 10; - init(s); + init(s, PCUObj); totW = 0; } double total() { @@ -26,12 +25,12 @@ namespace parma { ShapeTargets(); int smallLimit; double totW; - void init(Sides* s) { - const unsigned maxNb = TO_UINT(PCU_Max_Int(s->size())); + void init(Sides* s, pcu::PCU *PCUObj) { + const unsigned maxNb = TO_UINT(PCUObj->Max(s->size())); if( s->size() != maxNb ) return; - PCU_Debug_Print("maxNb %d\n", maxNb); + PCUObj->DebugPrint("maxNb %d\n", maxNb); std::string sstr = s->print("sides"); - PCU_Debug_Print("%s\n", sstr.c_str()); + PCUObj->DebugPrint("%s\n", sstr.c_str()); int small = INT_MAX; s->begin(); const Sides::Item* side; @@ -39,7 +38,7 @@ namespace parma { if( side->second < small ) small = side->second; s->end(); - PCU_Debug_Print("small %d\n", small); + PCUObj->DebugPrint("small %d\n", small); if( small > smallLimit ) return; s->begin(); while( (side = s->iterate()) ) @@ -47,10 +46,10 @@ namespace parma { set(side->first, small); s->end(); std::string tgtstr = print("targets"); - PCU_Debug_Print("%s\n", tgtstr.c_str()); + PCUObj->DebugPrint("%s\n", tgtstr.c_str()); } }; - Targets* makeShapeTargets(Sides* s) { - return new ShapeTargets(s); + Targets* makeShapeTargets(Sides* s, pcu::PCU *PCUObj) { + return new ShapeTargets(s, PCUObj); } } //end namespace diff --git a/parma/diffMC/parma_sides.cc b/parma/diffMC/parma_sides.cc index 68404a5bf..3febcbe23 100644 --- a/parma/diffMC/parma_sides.cc +++ b/parma/diffMC/parma_sides.cc @@ -1,8 +1,7 @@ -#include #include "parma_sides.h" #include "parma_convert.h" -double parma::avgSharedSides(parma::Sides* s) { +double parma::avgSharedSides(parma::Sides* s, pcu::PCU *PCUObj) { double tot = TO_DOUBLE(s->total()); - tot = PCU_Add_Double(tot); - return tot / PCU_Comm_Peers(); + tot = PCUObj->Add(tot); + return tot / PCUObj->Peers(); } diff --git a/parma/diffMC/parma_sides.h b/parma/diffMC/parma_sides.h index cf1ba2557..b22632a8d 100644 --- a/parma/diffMC/parma_sides.h +++ b/parma/diffMC/parma_sides.h @@ -16,7 +16,7 @@ namespace parma { Sides* makeElmSideSides(apf::Mesh* m); Sides* makeVtxSides(apf::Mesh* m); - double avgSharedSides(Sides* s); + double avgSharedSides(Sides* s, pcu::PCU *PCUObj); } #endif diff --git a/parma/diffMC/parma_step.cc b/parma/diffMC/parma_step.cc index 5064189d6..298e041e1 100644 --- a/parma/diffMC/parma_step.cc +++ b/parma/diffMC/parma_step.cc @@ -28,10 +28,10 @@ namespace parma { bool Stepper::step(double maxImb, int verbosity) { double imb, avg; - getImbalance(weights, imb, avg); + getImbalance(weights, imb, avg, m->getPCU()); if ( !m->getPCU()->Self() && verbosity ) status("%s imbalance %.3f avg %.3f\n", name, imb, avg); - if ( stop->stop(imb,maxImb) ) + if ( stop->stop(imb,maxImb,m->getPCU()) ) return false; apf::Migration* plan = selects->run(targets); int planSz = m->getPCU()->Add(plan->count()); diff --git a/parma/diffMC/parma_stop.cc b/parma/diffMC/parma_stop.cc index b110d340a..08b48b346 100644 --- a/parma/diffMC/parma_stop.cc +++ b/parma/diffMC/parma_stop.cc @@ -1,16 +1,15 @@ #include "parma_stop.h" #include "parma_commons.h" -#include #include namespace parma { BalOrStall::BalOrStall(Average* imb, Average* sides, double sidesTol, int v) : i(imb), s(sides), sTol(sidesTol), verbose(v) {} - bool BalOrStall::stop(double imb, double maxImb) { + bool BalOrStall::stop(double imb, double maxImb, pcu::PCU *PCUObj) { const double iTol = (maxImb-1)*.01; const double iSlope = i->avg(); const double sSlope = s->avg(); - if( !PCU_Comm_Self() && verbose ) + if( !PCUObj->Self() && verbose ) parmaCommons::status("imbSlope %f sidesSlope %f\n", iSlope, sSlope); return imb < maxImb || ( fabs(iSlope) < iTol && fabs(sSlope) < sTol ); diff --git a/parma/diffMC/parma_stop.h b/parma/diffMC/parma_stop.h index f392ebfe3..daf7f2031 100644 --- a/parma/diffMC/parma_stop.h +++ b/parma/diffMC/parma_stop.h @@ -2,17 +2,18 @@ #define PARMA_STOP_H #include "parma_monitor.h" +#include "PCUObj.h" namespace parma { class Stop { public: virtual ~Stop() {} - virtual bool stop(double imb, double maxImb)=0; + virtual bool stop(double imb, double maxImb, pcu::PCU *PCUObj)=0; }; class Less : public Stop { public: ~Less() {} - bool stop(double imb, double maxImb) { + bool stop(double imb, double maxImb, pcu::PCU *PCUObj) { return imb < maxImb; } }; @@ -20,7 +21,7 @@ namespace parma { public: BalOrStall(Average* imb, Average* sides, double sidesTol, int verbose=0); ~BalOrStall() {} - bool stop(double imb, double maxImb); + bool stop(double imb, double maxImb, pcu::PCU *PCUObj); private: Average* i; Average* s; diff --git a/parma/diffMC/parma_targets.h b/parma/diffMC/parma_targets.h index 93fd95f07..38f82f61d 100644 --- a/parma/diffMC/parma_targets.h +++ b/parma/diffMC/parma_targets.h @@ -22,7 +22,7 @@ namespace parma { double vtxTol, double alpha); Targets* makeElmLtVtxEdgeTargets(Sides* s, Weights* w[3], int sideTol, double vtxTol, double edgeTol, double alpha); - Targets* makeShapeTargets(Sides* s); + Targets* makeShapeTargets(Sides* s, pcu::PCU *PCUObj); Targets* makeGhostTargets(Sides* s, Weights* w, Ghosts* g, double alpha); } #endif diff --git a/parma/diffMC/parma_vtxBalancer.cc b/parma/diffMC/parma_vtxBalancer.cc index 599b39e1b..ec2525222 100644 --- a/parma/diffMC/parma_vtxBalancer.cc +++ b/parma/diffMC/parma_vtxBalancer.cc @@ -21,7 +21,7 @@ namespace { VtxBalancer(apf::Mesh* m, double f, int v) : Balancer(m, f, v, "vertices") { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); @@ -35,7 +35,7 @@ namespace { parma::Targets* t = parma::makeWeightSideTargets(s, w, sideTol, factor); parma::Selector* sel = parma::makeVtxSelector(mesh, wtag); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxVtxImb, iS, iA); monitorUpdate(avgSides, sS, sA); if( !mesh->getPCU()->Self() && verbose ) diff --git a/parma/diffMC/parma_vtxEdgeElmBalancer.cc b/parma/diffMC/parma_vtxEdgeElmBalancer.cc index f72230362..02b795c8c 100644 --- a/parma/diffMC/parma_vtxEdgeElmBalancer.cc +++ b/parma/diffMC/parma_vtxEdgeElmBalancer.cc @@ -25,7 +25,7 @@ namespace { status("maxVtx %.3f\n", maxVtx); } parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); @@ -38,7 +38,7 @@ namespace { const double maxEdgeImb = Parma_GetWeightedEntImbalance(mesh, wtag, 1); parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); parma::Weights* w[2] = {parma::makeEntWeights(mesh, wtag, s, 0), parma::makeEntWeights(mesh, wtag, s, 1)}; diff --git a/parma/diffMC/parma_vtxElmBalancer.cc b/parma/diffMC/parma_vtxElmBalancer.cc index ac0382474..90a96b14d 100644 --- a/parma/diffMC/parma_vtxElmBalancer.cc +++ b/parma/diffMC/parma_vtxElmBalancer.cc @@ -26,7 +26,7 @@ namespace { status("maxVtx %.3f\n", maxVtx); } parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); @@ -48,7 +48,7 @@ namespace { parma::Selector* sel = parma::makeElmLtVtxSelector(mesh, wtag, maxVtx); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); if( !mesh->getPCU()->Self() && verbose ) diff --git a/parma/diffMC/parma_vtxPtnWriter.cc b/parma/diffMC/parma_vtxPtnWriter.cc index 0c6e0f091..3a77013a9 100644 --- a/parma/diffMC/parma_vtxPtnWriter.cc +++ b/parma/diffMC/parma_vtxPtnWriter.cc @@ -28,9 +28,9 @@ namespace { if( m->getPCU()->Self() == m->getPCU()->Peers()-1 ) c += remainder; } - int getWriter(int id) { + int getWriter(int id, pcu::PCU *PCUObj) { int writer = id / pp; - if ( writer == PCU_Comm_Peers() ) + if ( writer == PCUObj->Peers() ) writer--; return writer; } @@ -52,7 +52,7 @@ namespace { while( (vtx = m->iterate(itr)) ) { if( parma::isOwned(m, vtx) ) { m->getIntTag(vtx, t, &id); - m->getPCU()->Pack(p.getWriter(id), id); + m->getPCU()->Pack(p.getWriter(id, m->getPCU()), id); } } m->end(itr); @@ -71,9 +71,9 @@ namespace { f << a[i] << '\n'; } - void open(const char* name, std::fstream& f) { + void open(const char* name, std::fstream& f, pcu::PCU *PCUObj) { std::stringstream ss; - ss << name << PCU_Comm_Self() << ".ptn"; + ss << name << PCUObj->Self() << ".ptn"; std::string s = ss.str(); f.open(s.c_str(), std::fstream::out); } @@ -81,7 +81,7 @@ namespace { void writeVtxPtn(apf::Mesh* m, const char* name) { PCU_ALWAYS_ASSERT(name); std::fstream f; - open(name,f); + open(name,f,m->getPCU()); Ptn p(m); int* ptn = new int[p.count()]; getPtnArray(m, p, ptn); diff --git a/parma/diffMC/parma_weights.h b/parma/diffMC/parma_weights.h index be028235d..40ddd299d 100644 --- a/parma/diffMC/parma_weights.h +++ b/parma/diffMC/parma_weights.h @@ -26,8 +26,8 @@ namespace parma { double getMaxWeight(apf::Mesh* m, apf::MeshTag* w, int entDim); double getAvgWeight(apf::Mesh* m, apf::MeshTag* w, int entDim); double getWeight(apf::Mesh* m, apf::MeshTag* w, int entDim); - double getMaxWeight(Weights* w); - void getImbalance(Weights* w, double& imb, double& avg); + double getMaxWeight(Weights* w, pcu::PCU *PCUObj); + void getImbalance(Weights* w, double& imb, double& avg, pcu::PCU *PCUObj); } #endif diff --git a/parma/parma.cc b/parma/parma.cc index 1e0d5c10d..724952182 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "diffMC/maximalIndependentSet/mis.h" @@ -89,24 +88,24 @@ namespace { } void getWeightedStats( - int dim, double (*loc)[4], double (*tot)[4], + pcu::PCU *PCUObj, int dim, double (*loc)[4], double (*tot)[4], double (*min)[4], double (*max)[4], double (*avg)[4]) { for(int d=0; d<=dim; d++) (*min)[d] = (*max)[d] = (*tot)[d] = (*loc)[d]; - PCU_Min_Doubles(*min, dim+1); - PCU_Max_Doubles(*max, dim+1); - PCU_Add_Doubles(*tot, dim+1); + PCUObj->Min(*min, dim+1); + PCUObj->Max(*max, dim+1); + PCUObj->Add(*tot, dim+1); for(int d=0; d<=dim; d++) { (*avg)[d] = (*tot)[d]; - (*avg)[d] /= TO_DOUBLE(PCU_Comm_Peers()); + (*avg)[d] /= TO_DOUBLE(PCUObj->Peers()); } } - void getStats(int& loc, long& tot, int& min, int& max, double& avg) { - min = PCU_Min_Int(loc); - max = PCU_Max_Int(loc); - tot = PCU_Add_Long(TO_LONG(loc)); - avg = TO_DOUBLE(tot) / PCU_Comm_Peers(); + void getStats(pcu::PCU *PCUObj, int& loc, long& tot, int& min, int& max, double& avg) { + min = PCUObj->Min(loc); + max = PCUObj->Max(loc); + tot = PCUObj->Add(TO_LONG(loc)); + avg = TO_DOUBLE(tot) / PCUObj->Peers(); } using parmaCommons::status; @@ -138,7 +137,7 @@ namespace { getPartWeights(m, w, &weight); double minEnt[4] = {0,0,0,0}, maxEnt[4] = {0,0,0,0}; double totEnt[4] = {0,0,0,0}, avgEnt[4] = {0,0,0,0}; - getWeightedStats(m->getDimension(), &weight, &totEnt, &minEnt, &maxEnt, &avgEnt); + getWeightedStats(m->getPCU(), m->getDimension(), &weight, &totEnt, &minEnt, &maxEnt, &avgEnt); const char* orders[4] = {"vtx","edge","face","rgn"}; if(!m->getPCU()->Self()) { for( int d=0; d<=m->getDimension(); d++) @@ -252,20 +251,20 @@ int Parma_GetSmallestSideMaxNeighborParts(apf::Mesh* m) { void Parma_GetOwnedBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, int& max, double& avg) { loc = numBdryVtx(m); - getStats(loc, tot, min, max, avg); + getStats(m->getPCU(), loc, tot, min, max, avg); } void Parma_GetSharedBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, int& max, double& avg) { bool onlyShared = true; loc = numBdryVtx(m,onlyShared); - getStats(loc, tot, min, max, avg); + getStats(m->getPCU(), loc, tot, min, max, avg); } void Parma_GetMdlBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, int& max, double& avg) { loc = numMdlBdryVtx(m); - getStats(loc, tot, min, max, avg); + getStats(m->getPCU(), loc, tot, min, max, avg); } void Parma_GetDisconnectedStats(apf::Mesh* m, int& max, double& avg, int& loc) { From 6cbd11de475054d6e027b8e2c9200885e36d5df1 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 27 Mar 2024 14:26:44 -0400 Subject: [PATCH 077/141] added switchPCU to apfMesh to use in parma_group.cc --- apf/apfMesh.cc | 6 ++++++ apf/apfMesh.h | 1 + parma/group/parma_group.cc | 20 ++++++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index fc20f2bdd..1338e0ab8 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -415,6 +415,12 @@ GlobalNumbering* Mesh::getGlobalNumbering(int i) return globalNumberings[i]; } +void Mesh::switchPCU(pcu::PCU *newPCU) +{ + PCU_ALWAYS_ASSERT(newPCU != nullptr); + pcu_ = newPCU; +} + void unite(Parts& into, Parts const& from) { diff --git a/apf/apfMesh.h b/apf/apfMesh.h index 772c5225e..c56ba26c5 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -399,6 +399,7 @@ class Mesh GlobalNumbering* getGlobalNumbering(int i); /** \brief get the global pcu */ pcu::PCU* getPCU() const {return pcu_;} + void switchPCU(pcu::PCU *newPCU); /** \brief true if any associated fields use array storage */ bool hasFrozenFields; protected: diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index 2afec2417..2ec4d2380 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -1,11 +1,12 @@ -#include +//#include #include +#include using apf::Remap; static void retreat(apf::Mesh2* m, Remap& remap) { - int to = remap(PCU_Comm_Self()); + int to = remap(m->getPCU()->Self()); apf::Migration* plan = new apf::Migration(m); apf::MeshIterator* it = m->begin(m->getDimension()); apf::MeshEntity* e; @@ -32,17 +33,20 @@ static void runInGroups( Remap& outMap, GroupCode& code) { - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); int groupRank = inMap(self); int group = groupMap(self); - MPI_Comm oldComm = PCU_Get_Comm(); + auto* expandedPCU = m->getPCU(); + //pcu::PCU *expandedPCU = m->getPCU(); MPI_Comm groupComm; - MPI_Comm_split(oldComm, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + MPI_Comm_split(expandedPCU->GetMPIComm(), group, groupRank, &groupComm); + auto groupedPCU = std::unique_ptr(new pcu::PCU(groupComm)); + m->switchPCU(groupedPCU.get()); + //PCU_Switch_Comm(groupComm); if (m) apf::remapPartition(m, inMap); code.run(group); - PCU_Switch_Comm(oldComm); + m->switchPCU(expandedPCU); MPI_Comm_free(&groupComm); if (m) apf::remapPartition(m, outMap); @@ -100,7 +104,7 @@ void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun) { apf::Modulo inMap(factor); apf::Divide groupMap(factor); - apf::Unmodulo outMap(PCU_Comm_Self(), factor); + apf::Unmodulo outMap(m->getPCU()->Self(), factor); runInGroups(m, inMap, groupMap, outMap, toRun); } From 000418ef62e355d144eeaa9c9f62b5689b645091 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 28 Mar 2024 10:41:11 -0400 Subject: [PATCH 078/141] parma_group.cc fixed to use switchPCU, may need to change later --- parma/group/parma_group.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index 2ec4d2380..0c0e9487b 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -1,4 +1,3 @@ -//#include #include #include @@ -37,12 +36,10 @@ static void runInGroups( int groupRank = inMap(self); int group = groupMap(self); auto* expandedPCU = m->getPCU(); - //pcu::PCU *expandedPCU = m->getPCU(); MPI_Comm groupComm; MPI_Comm_split(expandedPCU->GetMPIComm(), group, groupRank, &groupComm); auto groupedPCU = std::unique_ptr(new pcu::PCU(groupComm)); m->switchPCU(groupedPCU.get()); - //PCU_Switch_Comm(groupComm); if (m) apf::remapPartition(m, inMap); code.run(group); From bd265d8115d03a271f3c162b88c0e8bce841195e Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 28 Mar 2024 11:22:18 -0400 Subject: [PATCH 079/141] parallelReduce function fix --- apf/apf.h | 6 +++++- apf/apfIntegrate.cc | 4 ++-- sam/samElementCount.cc | 4 ++-- spr/sprEstimateError.cc | 4 ++-- spr/sprEstimateTargetError.cc | 4 ++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apf/apf.h b/apf/apf.h index 7db3f37a2..9a9ba849a 100644 --- a/apf/apf.h +++ b/apf/apf.h @@ -12,6 +12,10 @@ #include "apfNew.h" #include "apfDynamicArray.h" +namespace pcu{ + class PCU; +} + #include #include #include @@ -460,7 +464,7 @@ class Integrator * process-local integrations into a global mesh integration, * if that is the user's goal. */ - virtual void parallelReduce(); + virtual void parallelReduce(pcu::PCU*); protected: int order; int ipnode; diff --git a/apf/apfIntegrate.cc b/apf/apfIntegrate.cc index bb93ad051..2925524c1 100644 --- a/apf/apfIntegrate.cc +++ b/apf/apfIntegrate.cc @@ -637,7 +637,7 @@ void Integrator::outElement() { } -void Integrator::parallelReduce() +void Integrator::parallelReduce(pcu::PCU*) { } @@ -656,7 +656,7 @@ void Integrator::process(Mesh* m, int d) destroyMeshElement(e); } m->end(elements); - this->parallelReduce(); + this->parallelReduce(m->getPCU()); } void Integrator::process(MeshElement* e) diff --git a/sam/samElementCount.cc b/sam/samElementCount.cc index 024199d9c..43368c4f6 100644 --- a/sam/samElementCount.cc +++ b/sam/samElementCount.cc @@ -52,8 +52,8 @@ class TotalMetricVolumeIso : public apf::Integrator { double vhat = getVolumeChange(dim, h); sum += vhat * w * dV; } - virtual void parallelReduce() { - sum = PCU_Add_Double(sum); + virtual void parallelReduce(pcu::PCU *PCUObj) { + sum = PCUObj->Add(sum); } }; diff --git a/spr/sprEstimateError.cc b/spr/sprEstimateError.cc index 72c68f618..7aef35518 100644 --- a/spr/sprEstimateError.cc +++ b/spr/sprEstimateError.cc @@ -25,9 +25,9 @@ class SInt : public apf::Integrator SInt(int order): apf::Integrator(order),r(0) {} - void parallelReduce() + void parallelReduce(pcu::PCU *PCUObj) { - PCU_Add_Doubles(&r,1); + PCUObj->Add(&r,1); } void reset() {r=0;} double r; diff --git a/spr/sprEstimateTargetError.cc b/spr/sprEstimateTargetError.cc index 0b29d5467..38b4be2eb 100644 --- a/spr/sprEstimateTargetError.cc +++ b/spr/sprEstimateTargetError.cc @@ -66,9 +66,9 @@ class ScalarIntegrator : public apf::Integrator result(0) { } - void parallelReduce() + void parallelReduce(pcu::PCU *PCUObj) { - PCU_Add_Doubles(&result,1); + PCUObj->Add(&result,1); } double result; }; From 7f830669cc7fbbdf900732c79435a21c9b0d5916 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:40:50 -0400 Subject: [PATCH 080/141] crv corrected makeEmptyMdsMesh use --- crv/crvVtk.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc index 5258aca1b..73dcacf30 100644 --- a/crv/crvVtk.cc +++ b/crv/crvVtk.cc @@ -948,7 +948,7 @@ static void makeDirectories(const char* prefix, int type, int n) void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix) { - apf::Mesh2* wireMesh = apf::makeEmptyMdsMesh(NULL, 1, false); + apf::Mesh2* wireMesh = apf::makeEmptyMdsMesh(NULL, 1, false, m->getPCU()); apf::Field* f = m->getCoordinateField(); apf::MeshEntity* ent; apf::MeshIterator* it; From 8469c37f3d0955f5b5707a8369f2b26e470416d3 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:41:12 -0400 Subject: [PATCH 081/141] ma corrected makeEmptyMdsMesh use --- ma/maDBG.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ma/maDBG.cc b/ma/maDBG.cc index 8ed629fbe..2460ec809 100644 --- a/ma/maDBG.cc +++ b/ma/maDBG.cc @@ -223,7 +223,7 @@ void createCavityMesh(ma::Adapt* a, ma::Mesh* m = a->mesh; gmi_register_null(); - ma::Mesh* cavityMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 3, false); + ma::Mesh* cavityMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 3, false, m->getPCU()); size_t n = tets.getSize(); for (size_t i = 0; i < n; ++i) { @@ -364,7 +364,7 @@ void visualizeSizeField( const char* outputPrefix) { // create the size-field visualization mesh - apf::Mesh2* msf = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); + apf::Mesh2* msf = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, m->getPCU()); apf::FieldShape* fs = apf::getShape(sizes); int dim = m->getDimension(); From 2b90da5a9dbfd7623cf0467de57b42b69c9bc030 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:43:01 -0400 Subject: [PATCH 082/141] fixed BoxBuilder class and loadMdsFromANSYS to take an optional PCUObj --- mds/apfBox.cc | 10 +++++----- mds/apfBox.h | 6 +++--- mds/apfMDS.h | 2 +- mds/mdsANSYS.cc | 18 ++++++++++++++---- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/mds/apfBox.cc b/mds/apfBox.cc index b6dec78c0..a8b281009 100644 --- a/mds/apfBox.cc +++ b/mds/apfBox.cc @@ -67,7 +67,7 @@ int Grid::in(Indices is) BoxBuilder::BoxBuilder(int nx, int ny, int nz, double wx, double wy, double wz, - bool is): + bool is, pcu::PCU *PCUObj): grid(nx + 1, ny + 1, nz + 1), mgrid(nx ? 3 : 1, ny ? 3 : 1, nz ? 3 : 1) { @@ -329,17 +329,17 @@ void BoxBuilder::buildMeshAndModel() Mesh2* makeMdsBox( int nex, int ney, int nez, - double wx, double wy, double wz, bool is) + double wx, double wy, double wz, bool is, pcu::PCU *PCUObj) { - BoxBuilder bb(nex, ney, nez, wx, wy, wz, is); + BoxBuilder bb(nex, ney, nez, wx, wy, wz, is, PCUObj); return bb.m; } gmi_model* makeMdsBoxModel( int nex, int ney, int nez, - double wx, double wy, double wz, bool is) + double wx, double wy, double wz, bool is, pcu::PCU *PCUObj) { - BoxBuilder bb(nex, ney, nez, wx, wy, wz, is); + BoxBuilder bb(nex, ney, nez, wx, wy, wz, is, PCUObj); return bb.buildModel(); } diff --git a/mds/apfBox.h b/mds/apfBox.h index 035a29b78..5fa443ef7 100644 --- a/mds/apfBox.h +++ b/mds/apfBox.h @@ -39,7 +39,7 @@ struct BoxBuilder std::vector v; BoxBuilder(int nx, int ny, int nz, double wx, double wy, double wz, - bool is); + bool is, pcu::PCU *PCUObj); void formModelTable(); void addModelUse(gmi_base* gb, agm_bdry ab, Indices di); gmi_model* buildModel(); @@ -71,11 +71,11 @@ struct BoxBuilder \param is true = simplical mesh, false = quad/hex \details set ny,nz=0 for a 1D mesh, set nz=0 for a 2D mesh */ Mesh2* makeMdsBox( - int nx, int ny, int nz, double wx, double wy, double wz, bool is); + int nx, int ny, int nz, double wx, double wy, double wz, bool is, pcu::PCU *PCUObj); /** \brief see makeMdsBox - only creates geometric model */ gmi_model* makeMdsBoxModel( - int nx, int ny, int nz, double wx, double wy, double wz, bool is); + int nx, int ny, int nz, double wx, double wy, double wz, bool is, pcu::PCU *PCUObj); } diff --git a/mds/apfMDS.h b/mds/apfMDS.h index 6caea293b..2d14e9917 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -238,7 +238,7 @@ void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile currently, ANSYS element types SOLID72 and SOLID92 are supported, which become linear and quadratic tetrahedra, respectively. */ -Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile); +Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PCUObj = nullptr); void disownMdsModel(Mesh2* in); diff --git a/mds/mdsANSYS.cc b/mds/mdsANSYS.cc index e27961bfc..894d96953 100644 --- a/mds/mdsANSYS.cc +++ b/mds/mdsANSYS.cc @@ -90,7 +90,7 @@ static void parseNodes(const char* nodefile, Nodes& nodes) typedef std::map Vertices; -static Mesh2* parseElems(const char* elemfile, Nodes& nodes) +static Mesh2* parseElems(const char* elemfile, Nodes& nodes, pcu::PCU *PCUObj = nullptr) { Mesh2* m = 0; std::ifstream f(elemfile); @@ -107,7 +107,12 @@ static Mesh2* parseElems(const char* elemfile, Nodes& nodes) apf::Numbering* enumbers = 0; while (parseElem(f, en, type, id, shape)) { if (!m) { - m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false); + if(PCUObj != nullptr){ + m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false, PCUObj); + } else { + m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false); + } + if (shape != m->getShape()) changeMeshShape(m, shape, false); enumbers = createNumbering(m, "ansys_element", getConstant(m->getDimension()), 1); @@ -151,11 +156,16 @@ static Mesh2* parseElems(const char* elemfile, Nodes& nodes) return m; } -Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile) +Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PCUObj) { Nodes nodes; parseNodes(nodefile, nodes); - Mesh2* m = parseElems(elemfile, nodes); + Mesh2* m; + if(PCUObj != nullptr){ + m = parseElems(elemfile, nodes, PCUObj); + } else { + m = parseElems(elemfile, nodes); + } m->acceptChanges(); deriveMdsModel(m); return m; From 961f09a647285d9fcd01940366c3d16ccb276af1 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:47:26 -0400 Subject: [PATCH 083/141] got rid of undef PCU_FORMAT_ATTRIBUTE, fixed compilation errors that would come up --- pcu/PCUObj.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index fd3e4f84b..723b6c048 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -1,5 +1,6 @@ #ifndef SCOREC_PCU_PCUOBJ_H #define SCOREC_PCU_PCUOBJ_H + #include #include #include "pcu_defines.h" @@ -125,7 +126,6 @@ PCU_EXPL_INST_DECL(double) } // namespace pcu -#undef PCU_FORMAT_ATTRIBUTE -#endif // SCOREC_PCU_PCUOBJ_H +#endif // PCUOBJ_H From daed911a580a9d187f8784383791de79693594b4 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:48:33 -0400 Subject: [PATCH 084/141] phasta - fixed createMdsMesh use --- phasta/ph_convert.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 1fca8abaa..16b9bc3f2 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -255,7 +255,7 @@ int main(int argc, char** argv) if (should_attach_order) attachOrder(simApfMesh); ph::buildMapping(simApfMesh); - apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh, pcu_obj.get()); + apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh); double t3 = pcu::Time(); if(!mesh->getPCU()->Self()) lion_eprint(1, "created the apf_mds mesh in %f seconds\n", t3-t2); From 946ba7ce370de269fce4c2f4a9d04cc5ca0a3e09 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:49:04 -0400 Subject: [PATCH 085/141] got rid of an #include PCU.h --- pumi/pumi_gentity.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/pumi/pumi_gentity.cc b/pumi/pumi_gentity.cc index 28a6df473..eabf504e6 100644 --- a/pumi/pumi_gentity.cc +++ b/pumi/pumi_gentity.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include From 8c28b15d88c4b30f16693cfbec5cfec67c36762a Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:49:27 -0400 Subject: [PATCH 086/141] #include PCU.h deletion --- ree/ree.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ree/ree.h b/ree/ree.h index 5e337484c..93c6147ff 100644 --- a/ree/ree.h +++ b/ree/ree.h @@ -15,7 +15,6 @@ #include "apf.h" #include -#include #include #include "apfShape.h" #include "apfField.h" From 7a99da1455aed8fde07cecd7f6d98fc484a37a75 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:49:44 -0400 Subject: [PATCH 087/141] spr folder fixes --- spr/spr.h | 5 ++++- spr/sprEstimateError.cc | 5 ++--- spr/sprEstimateTargetError.cc | 5 ++--- spr/sprGetGradIPField.cc | 1 + 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/spr/spr.h b/spr/spr.h index 19cae047c..cd5a2007f 100644 --- a/spr/spr.h +++ b/spr/spr.h @@ -16,6 +16,8 @@ #include "apfNew.h" #include "apfDynamicVector.h" #include "apfDynamicMatrix.h" +#include "PCUObj.h" + /** \namespace spr * \brief All SPR error estimator functions @@ -43,7 +45,7 @@ apf::Field* recoverField(apf::Field* ip_field); * scales the output size field. * @returns a scalar mesh size field defined at vertices */ -apf::Field* getSPRSizeField(apf::Field* f, double adapt_ratio); +apf::Field* getSPRSizeField(apf::Field* f, double adapt_ratio, pcu::PCU *PCUObj); /** @brief run the SPR ZZ error estimator with a target # of output elems * @param f the integration-point input field @@ -55,6 +57,7 @@ apf::Field* getSPRSizeField(apf::Field* f, double adapt_ratio); apf::Field* getTargetSPRSizeField( apf::Field* f, size_t t, + pcu::PCU *PCUObj, double alpha=0.25, double beta=2.0); diff --git a/spr/sprEstimateError.cc b/spr/sprEstimateError.cc index 7aef35518..c80a16cb0 100644 --- a/spr/sprEstimateError.cc +++ b/spr/sprEstimateError.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include "spr.h" @@ -309,14 +308,14 @@ static void estimateError(Estimation* e) apf::destroyField(e->element_size); } -apf::Field* getSPRSizeField(apf::Field* eps, double adaptRatio) +apf::Field* getSPRSizeField(apf::Field* eps, double adaptRatio, pcu::PCU *PCUObj) { double t0 = pcu::Time(); Estimation e; setupEstimation(&e, eps, adaptRatio); estimateError(&e); double t1 = pcu::Time(); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_eprint(1,"SPR: error estimated in %f seconds\n",t1-t0); return e.size; } diff --git a/spr/sprEstimateTargetError.cc b/spr/sprEstimateTargetError.cc index 38b4be2eb..698a600a3 100644 --- a/spr/sprEstimateTargetError.cc +++ b/spr/sprEstimateTargetError.cc @@ -11,8 +11,6 @@ and engineering 195.48 (2006): 6626-6645. */ #include "spr.h" - -#include #include #include #include @@ -243,6 +241,7 @@ static void estimateError(Estimation* e) apf::Field* getTargetSPRSizeField( apf::Field* eps, size_t target, + pcu::PCU *PCUObj, double alpha, double beta) { @@ -253,7 +252,7 @@ apf::Field* getTargetSPRSizeField( target::setupEstimation(&e, eps, target, alpha, beta); target::estimateError(&e); double t1 = pcu::Time(); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_eprint(1, "SPR (target): error estimated in %f seconds\n",t1-t0); return e.vtx_size; } diff --git a/spr/sprGetGradIPField.cc b/spr/sprGetGradIPField.cc index 94be3d4d9..af4e49935 100644 --- a/spr/sprGetGradIPField.cc +++ b/spr/sprGetGradIPField.cc @@ -10,6 +10,7 @@ #include "apfShape.h" #include + namespace spr { apf::Field* getGradIPField(apf::Field* f, const char* name, int order) From 99ad2db4d28ca34ab16aa67065b9c5233548d767 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:00 -0400 Subject: [PATCH 088/141] 1d test file --- test/1d.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/1d.cc b/test/1d.cc index d43b828e7..0e823f477 100644 --- a/test/1d.cc +++ b/test/1d.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,12 +6,13 @@ #include #include #include +#include -void createMesh(gmi_model*& g, apf::Mesh2*& m, int n) +void createMesh(gmi_model*& g, apf::Mesh2*& m, int n, pcu::PCU *PCUObj) { gmi_register_null(); g = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(g, 1, false); + m = apf::makeEmptyMdsMesh(g, 1, false, PCUObj); apf::ModelEntity* left = m->findModelEntity(0, 0); apf::ModelEntity* right = m->findModelEntity(0, 1); apf::ModelEntity* mid = m->findModelEntity(1, 1); @@ -66,19 +66,20 @@ int main(int argc, char** argv) * 3 = mesh output name **/ PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_model* g; apf::Mesh2* m; int nverts = atoi(argv[1]); PCU_ALWAYS_ASSERT(2 <= nverts); PCU_ALWAYS_ASSERT(nverts <= 1000); - createMesh(g,m,nverts); + createMesh(g,m,nverts,pcu_obj.get()); test(m); gmi_write_dmg(g,argv[2]); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } From 77cb68f53734a0f3a971051db34648e5153f7561 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:07 -0400 Subject: [PATCH 089/141] align test file --- test/align.cc | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/align.cc b/test/align.cc index 8cca65a43..ed334a7e9 100644 --- a/test/align.cc +++ b/test/align.cc @@ -2,18 +2,18 @@ #include #include #include -#include #include #include #include +#include -void testTriEdge() +void testTriEdge(pcu::PCU *PCUObj) { int which, rotate; bool flip; for(int ed = 0; ed < 6; ++ed){ gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[3]; for (int i = 0; i < 3; ++i) v[i] = m->createVert(0); @@ -32,13 +32,13 @@ void testTriEdge() apf::destroyMesh(m); } } -void testTetEdge() +void testTetEdge(pcu::PCU *PCUObj) { int which, rotate; bool flip; for(int ed = 0; ed < 12; ++ed){ gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* v[4]; for (int i = 0; i < 4; ++i) v[i] = m->createVert(0); @@ -57,7 +57,7 @@ void testTetEdge() apf::destroyMesh(m); } } -void testTetTri() +void testTetTri(pcu::PCU *PCUObj) { int which, rotate; bool flip; @@ -65,7 +65,7 @@ void testTetTri() for(int flipped = 0; flipped < 2; ++flipped){ for(int fa = 0; fa < 12; ++fa){ gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* v[4]; for (int i = 0; i < 4; ++i) v[i] = m->createVert(0); @@ -91,12 +91,13 @@ void testTetTri() int main() { MPI_Init(0,0); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); - testTriEdge(); - testTetEdge(); - testTetTri(); - PCU_Comm_Free(); + testTriEdge(pcu_obj.get()); + testTetEdge(pcu_obj.get()); + testTetTri(pcu_obj.get()); + } MPI_Finalize(); } From 1dc3a945d6983f9ab49f2604a0f1ad59bd934483 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:17 -0400 Subject: [PATCH 090/141] aniso_ma_test file --- test/aniso_ma_test.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/aniso_ma_test.cc b/test/aniso_ma_test.cc index 4eb9a0278..763f72a83 100644 --- a/test/aniso_ma_test.cc +++ b/test/aniso_ma_test.cc @@ -3,9 +3,9 @@ #include #include #include -#include #include #include +#include #include @@ -46,10 +46,11 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; bool logInterpolation = atoi(argv[3]) > 0 ? true : false; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); apf::writeVtkFiles("aniso_before",m); AnIso sf(m); @@ -67,7 +68,7 @@ int main(int argc, char** argv) apf::writeVtkFiles("aniso_after",m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } From becd83ad0c7b961a61780732c5df6b7f4716e19d Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:25 -0400 Subject: [PATCH 091/141] ansys test file --- test/ansys.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/ansys.cc b/test/ansys.cc index c7c41e64a..57d82fbeb 100644 --- a/test/ansys.cc +++ b/test/ansys.cc @@ -2,29 +2,30 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); - apf::Mesh2* m = apf::loadMdsFromANSYS(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsFromANSYS(argv[1], argv[2], pcu_obj.get()); m->verify(); gmi_write_dmg(m->getModel(), argv[3]); m->writeNative(argv[4]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } From e591d9a5fc16da6a4d1f1c2ea5cd150b7636e6a0 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:35 -0400 Subject: [PATCH 092/141] assert_timing test file --- test/assert_timing.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/assert_timing.cc b/test/assert_timing.cc index 4692dd830..f65dcd52c 100644 --- a/test/assert_timing.cc +++ b/test/assert_timing.cc @@ -1,28 +1,29 @@ -#include +#include #include #include #include #include #include #include +#include double check_c_assert() { - double t0 = PCU_Time(); + double t0 = pcu::Time(); for (int i = 0; i < 100000000; ++i) { assert(pow((double)i, 1.01) < pow((double)i +1.0, 1.02)); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1-t0; } double check_pcu_assert() { - double t0 = PCU_Time(); + double t0 = pcu::Time(); for (int i = 0; i < 100000000; ++i) { double j = (double)i; double k = (double)i + 1.0; PCU_ALWAYS_ASSERT(pow(j, 1.01) < pow(k, 1.02)); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1-t0; } @@ -30,7 +31,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 2); int opt = atoi(argv[1]); MPI_Init(0,0); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); /* i'm avoiding conditionals inside for loops b/c i'm paranoid about the timings even though timings @@ -41,7 +43,7 @@ int main(int argc, char** argv) { else for (int i = 0; i < 5; ++i) printf("pcu assert in %f seconds\n", check_pcu_assert()); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } From dc48ea742b05d7adaa3a1f96579f4895b85a36a6 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:42 -0400 Subject: [PATCH 093/141] balance test file --- test/balance.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/balance.cc b/test/balance.cc index 8b46af5a1..cb6b2814c 100644 --- a/test/balance.cc +++ b/test/balance.cc @@ -3,19 +3,20 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); apf::MeshTag* weights = Parma_WeighByMemory(m); double step = 0.1; int verbose=1; apf::Balancer* balancer = Parma_MakeCentroidDiffuser(m, step, verbose); @@ -27,6 +28,6 @@ int main(int argc, char** argv) // destroy mds m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } From 462440854690d67925a0f42c6a6ad2975484fd77 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:50:53 -0400 Subject: [PATCH 094/141] bezierElevation test file --- test/bezierElevation.cc | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/test/bezierElevation.cc b/test/bezierElevation.cc index 8c5e1386b..6187fc696 100644 --- a/test/bezierElevation.cc +++ b/test/bezierElevation.cc @@ -6,11 +6,11 @@ #include #include #include -#include #include #include #include #include +#include /* * This contains all the tests for bezier elevation @@ -131,10 +131,10 @@ gmi_model* makeFaceModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeFaceModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[3], *edges[3]; apf::Vector3 points2D[3] = {apf::Vector3(0,0,0),apf::Vector3(1,0,0),apf::Vector3(1,1,0)}; @@ -162,10 +162,10 @@ static apf::Vector3 points3D[4] = apf::Vector3(0,0,1)}; -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TET,points3D); apf::deriveMdsModel(m); @@ -175,12 +175,12 @@ apf::Mesh2* createMesh3D() return m; } -void testEdgeElevation() +void testEdgeElevation(pcu::PCU *PCUObj) { for (int o = 1; o <= 5; ++o){ gmi_model* model = makeEdgeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false, PCUObj); apf::Vector3 points[2] = {apf::Vector3(0,0,0),apf::Vector3(1,1,1)}; apf::MeshEntity* v[2]; @@ -233,12 +233,12 @@ void testEdgeElevation() } } -void testTriElevation() +void testTriElevation(pcu::PCU *PCUObj) { for (int o = 1; o <= 5; ++o){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,o,0); bc.run(); apf::MeshIterator* it = m->begin(2); @@ -304,13 +304,13 @@ void testTriElevation() } } -void testTetElevation() +void testTetElevation(pcu::PCU *PCUObj) { gmi_register_null(); for (int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -399,11 +399,12 @@ void testTetElevation() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - testEdgeElevation(); - testTriElevation(); - testTetElevation(); - PCU_Comm_Free(); + testEdgeElevation(pcu_obj.get()); + testTriElevation(pcu_obj.get()); + testTetElevation(pcu_obj.get()); + } MPI_Finalize(); } From 0a7ccec30dc6fd0a6ed9793bb9664493f18073aa Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:51:00 -0400 Subject: [PATCH 095/141] box test file --- test/box.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/box.cc b/test/box.cc index e317232d4..fc0999f64 100644 --- a/test/box.cc +++ b/test/box.cc @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include namespace { @@ -20,11 +20,11 @@ bool is = true; const char* modelFile = 0; const char* meshFile = 0; -void verifyArgs(int argc, char** argv) +void verifyArgs(int argc, char** argv, pcu::PCU *PCUObj) { if (argc != 10) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) { printf("Usage: %s " " \n", argv[0]); @@ -61,23 +61,24 @@ void getArgs(char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - verifyArgs(argc, argv); - if(PCU_Comm_Peers()>1) { - if(PCU_Comm_Self()) + verifyArgs(argc, argv, pcu_obj.get()); + if(pcu_obj.get()->Peers()>1) { + if(pcu_obj.get()->Self()) fprintf(stderr, "%s must be run on a single process!\n", argv[0]); exit(1); } getArgs(argv); gmi_register_mesh(); - apf::Mesh2* m = apf::makeMdsBox(nx,ny,nz,wx,wy,wz,is); + apf::Mesh2* m = apf::makeMdsBox(nx,ny,nz,wx,wy,wz,is,pcu_obj.get()); gmi_model* g = m->getModel(); m->verify(); m->writeNative(meshFile); gmi_write_dmg(g, modelFile); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } From 54cc6be5ee00f78a976738629b42f9e11440601c Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:51:08 -0400 Subject: [PATCH 096/141] collapse test file --- test/collapse.cc | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/test/collapse.cc b/test/collapse.cc index da45c1fd4..eac060ec4 100644 --- a/test/collapse.cc +++ b/test/collapse.cc @@ -2,7 +2,8 @@ #include #include #include -#include +//#include +#include #include #ifdef HAVE_SIMMETRIX #include @@ -11,6 +12,7 @@ #include #include #include +#include namespace { @@ -26,10 +28,10 @@ namespace { } }; - void getConfig(int argc, char** argv) + void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: mpirun -np %s \n" "Reduce the part count of mesh from inPartCount to inPartCount/factor.\n", argv[0]); @@ -40,25 +42,26 @@ namespace { meshFile = argv[2]; outFile = argv[3]; partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj->Peers()); } } int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - PCU_Protect(); + pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,pcu_obj.get()); GroupCode code; - code.mesh = apf::loadMdsMesh(modelFile, meshFile); - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + code.mesh = apf::loadMdsMesh(modelFile, meshFile, pcu_obj.get()); + apf::Unmodulo outMap(pcu_obj.get()->Self(), pcu_obj.get()->Peers()); Parma_ShrinkPartition(code.mesh, partitionFactor, code); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); @@ -66,6 +69,6 @@ int main(int argc, char** argv) { gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } From f6e5d7c278a3a277bba99639c0349d5a4ae907e7 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Apr 2024 10:51:18 -0400 Subject: [PATCH 097/141] poisson test file --- test/poisson.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/poisson.cc b/test/poisson.cc index 62008c5d2..281d6f194 100644 --- a/test/poisson.cc +++ b/test/poisson.cc @@ -57,8 +57,8 @@ class Poisson { apf::destroyMesh(mesh); } - void run() { - setup_grid(); + void run(pcu::PCU *PCUObj) { + setup_grid(pcu::PCU *PCUObj); setup_lin_alg(); fill_volumetric(); fill_boundary(); @@ -95,14 +95,14 @@ class Poisson { apf::NewArray GBF; apf::NewArray numbers; - void setup_grid() { + void setup_grid(pcu::PCU *PCUObj) { int nx = num_grid; int ny = (num_dims > 1) ? num_grid : 0; int nz = (num_dims > 2) ? num_grid : 0; double wx = 1.0; double wy = (num_dims > 1) ? 1.0 : 0.0; double wz = (num_dims > 2) ? 1.0 : 0.0; - mesh = apf::makeMdsBox(nx, ny, nz, wx, wy, wz, true); + mesh = apf::makeMdsBox(nx, ny, nz, wx, wy, wz, true, PCUObj); apf::reorderMdsMesh(mesh); shape = apf::getLagrange(p_order); sol = apf::createField(mesh, "u", apf::SCALAR, shape); @@ -237,12 +237,12 @@ class Poisson { }; -void test(int dim, int p) { +void test(int dim, int p, pcu::PCU *PCUObj) { int steps = 6-dim; int n_grid = 6-dim; for (int i=0; i < steps; ++i) { Poisson poisson(dim, p, n_grid); - poisson.run(); + poisson.run(PCUObj); n_grid *= 2; } } @@ -252,12 +252,13 @@ void test(int dim, int p) { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(! PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(! pcu_obj.get()->Self()); int dim = atoi(argv[1]); int p = atoi(argv[2]); - test(dim, p); - PCU_Comm_Free(); + test(dim, p, pcu_obj.get()); + } MPI_Finalize(); } From 5f0ef7e38d27abfe9ee59abca99d82fd079b63a6 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 9 Apr 2024 15:00:05 -0400 Subject: [PATCH 098/141] mdsCGNS PCUHandle fix --- mds/mdsCGNS.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index 59e7df0d2..cd41abb5e 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -1076,7 +1076,7 @@ apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSB const int readDim = cellDim; // Salome cgns is a bit on the odd side: cellDim, physDim, ncoords are not always consistent - apf::Mesh2 *mesh = apf::makeEmptyMdsMesh(g, cellDim, false); + apf::Mesh2 *mesh = apf::makeEmptyMdsMesh(g, cellDim, false, static_cast(h.ptr)); apf::GlobalToVert globalToVert; LocalElementRanges localElementRanges; From 6452f43b89753eed2555a16bdb7be67d1f0f33be Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 9 Apr 2024 15:00:52 -0400 Subject: [PATCH 099/141] maximal independent set fixes --- .../maximalIndependentSet/mersenne_twister.cc | 8 +- .../maximalIndependentSet/mersenne_twister.h | 4 +- parma/diffMC/maximalIndependentSet/mis.h | 4 +- parma/diffMC/maximalIndependentSet/misLuby.cc | 144 +++++++++--------- .../maximalIndependentSet/test/testMIS.cc | 71 ++++----- parma/diffMC/parma_dcpartFixer.cc | 2 +- parma/parma.cc | 2 +- parma/parma.h | 4 +- 8 files changed, 121 insertions(+), 118 deletions(-) diff --git a/parma/diffMC/maximalIndependentSet/mersenne_twister.cc b/parma/diffMC/maximalIndependentSet/mersenne_twister.cc index 613e08ff1..25172da7a 100644 --- a/parma/diffMC/maximalIndependentSet/mersenne_twister.cc +++ b/parma/diffMC/maximalIndependentSet/mersenne_twister.cc @@ -34,8 +34,8 @@ static unsigned long mt[N]; /* the array for the state vector */ static int mti = N + 1; /* mti == N + 1 means mt is not initialized */ namespace { - void fail(const char* msg) { - if( ! PCU_Comm_Self() ) + void fail(const char* msg, pcu::PCU *PCUObj) { + if( ! PCUObj->Self() ) lion_eprint(1, "%s", msg); exit(EXIT_FAILURE); } @@ -53,14 +53,14 @@ void mersenne_twister_seed(unsigned seed) mt[mti] = (6909 * mt[mti - 1]) & 0xFFFFFFFF; } -unsigned mersenne_twister(void) +unsigned mersenne_twister(pcu::PCU *PCUObj) { static unsigned long const mag01[2] = {0x0, MATRIX_A}; /* mag01[x] = x * MATRIX_A fox x = {0,1} */ unsigned long y; int kk; if (mti == N + 1) - fail("mersenne twister was not seeded before use\n"); + fail("mersenne twister was not seeded before use\n", PCUObj); if (mti == N) { /* generate N words at one time */ for (kk = 0; kk < N - M; ++kk) { y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK); diff --git a/parma/diffMC/maximalIndependentSet/mersenne_twister.h b/parma/diffMC/maximalIndependentSet/mersenne_twister.h index 235c4ef62..66d82776c 100644 --- a/parma/diffMC/maximalIndependentSet/mersenne_twister.h +++ b/parma/diffMC/maximalIndependentSet/mersenne_twister.h @@ -1,7 +1,9 @@ #ifndef MERSENNE_TWISTER_H #define MERSENNE_TWISTER_H +#include + void mersenne_twister_seed(unsigned seed); -unsigned mersenne_twister(void); +unsigned mersenne_twister(pcu::PCU *PCUObj); #endif diff --git a/parma/diffMC/maximalIndependentSet/mis.h b/parma/diffMC/maximalIndependentSet/mis.h index 2c760435b..23165fa83 100644 --- a/parma/diffMC/maximalIndependentSet/mis.h +++ b/parma/diffMC/maximalIndependentSet/mis.h @@ -10,7 +10,7 @@ #include "mpi.h" #include -#include "PCU.h" +#include "PCUObj.h" #define MIS_ITERATE(t,w,i) \ for (t::iterator i = (w).begin(); \ @@ -52,7 +52,7 @@ namespace misLuby { * @param randNumsPredefined (In) 0: compute random numbers, 1:uses defined random numbers * @return 1 if local part is in mis, 0 o.w. */ -int mis(misLuby::partInfo& part, +int mis(misLuby::partInfo& part, pcu::PCU *PCUObj, bool randNumsPredefined = false, bool isNeighbors = false); diff --git a/parma/diffMC/maximalIndependentSet/misLuby.cc b/parma/diffMC/maximalIndependentSet/misLuby.cc index b2b2fbf5e..896f19c6e 100644 --- a/parma/diffMC/maximalIndependentSet/misLuby.cc +++ b/parma/diffMC/maximalIndependentSet/misLuby.cc @@ -35,12 +35,12 @@ using std::set; using namespace misLuby; namespace { - int generateRandomNumber() { - return (int) mersenne_twister(); + int generateRandomNumber(pcu::PCU *PCUObj) { + return (int) mersenne_twister(PCUObj); } - void setRandomNum(partInfo& part) { - part.randNum = generateRandomNumber(); + void setRandomNum(partInfo& part, pcu::PCU *PCUObj) { + part.randNum = generateRandomNumber(PCUObj); // don't select self nets until all other nets are selected if ( 1 == part.net.size() ) part.randNum = std::numeric_limits::max(); @@ -52,31 +52,31 @@ namespace { p.netAdjParts.erase(*nodeItr); } - int sendAdjNetsToNeighbors(partInfo& part, vector& nbNet) { - PCU_Comm_Begin(); + int sendAdjNetsToNeighbors(partInfo& part, vector& nbNet, pcu::PCU *PCUObj) { + PCUObj->Begin(); MIS_ITERATE(vector, part.adjPartIds, adjPartIdItr) { const int destRank = *adjPartIdItr; MIS_ITERATE(vector, nbNet, nbItr) { if (nbItr->partId != *adjPartIdItr) { //pack destination part Id - PCU_COMM_PACK(destRank, *adjPartIdItr); + PCUObj->Pack(destRank, *adjPartIdItr); //adjacent part Id - PCU_COMM_PACK(destRank, nbItr->partId); + PCUObj->Pack(destRank, nbItr->partId); //adjacent part's random number - PCU_COMM_PACK(destRank, nbItr->randNum); + PCUObj->Pack(destRank, nbItr->randNum); //net size size_t n = nbItr->net.size(); - PCU_COMM_PACK(destRank, n); + PCUObj->Pack(destRank, n); //net for (vector::iterator pItr = nbItr->net.begin(); pItr != nbItr->net.end(); pItr++) { - PCU_COMM_PACK( destRank, *pItr); + PCUObj->Pack( destRank, *pItr); } } } @@ -84,124 +84,124 @@ namespace { return 0; } - void sendIntsToNeighbors(partInfo& part, vector& msg, int tag) { - PCU_Comm_Begin(); + void sendIntsToNeighbors(partInfo& part, vector& msg, int tag, pcu::PCU *PCUObj) { + PCUObj->Begin(); MIS_ITERATE(vector, part.adjPartIds, adjPartIdItr) { const int destRank = *adjPartIdItr; //pack msg tag - PCU_COMM_PACK(destRank, tag); + PCUObj->Pack(destRank, tag); //pack destination part Id - PCU_COMM_PACK(destRank, *adjPartIdItr); + PCUObj->Pack(destRank, *adjPartIdItr); //pack array length size_t n = msg.size(); - PCU_COMM_PACK(destRank, n); + PCUObj->Pack(destRank, n); //pack int array for ( vector::iterator pItr = msg.begin(); pItr != msg.end(); pItr++ ) { - PCU_COMM_PACK(destRank, *pItr); + PCUObj->Pack(destRank, *pItr); } } } - void unpackInts(vector& msg, int tag) { - const int rank = PCU_Comm_Self(); + void unpackInts(vector& msg, int tag, pcu::PCU *PCUObj) { + const int rank = PCUObj->Self(); //unpack msg tag int inTag; - PCU_COMM_UNPACK(inTag); + PCUObj->Unpack(inTag); MIS_FAIL_IF(tag != inTag, "tags do not match"); //unpack destination part Id int destPartId; - PCU_COMM_UNPACK(destPartId); + PCUObj->Unpack(destPartId); PCU_ALWAYS_ASSERT(rank == destPartId); //unpack array length size_t n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); //unpack int array int buff; for(size_t i=0; i < n; i++) { - PCU_COMM_UNPACK(buff); + PCUObj->Unpack(buff); msg.push_back(buff); } } - void recvIntsFromNeighbors(vector& msg, int tag) { - while(PCU_Comm_Listen()) - unpackInts(msg, tag); + void recvIntsFromNeighbors(vector& msg, int tag, pcu::PCU *PCUObj) { + while(PCUObj->Listen()) + unpackInts(msg, tag, PCUObj); } - int sendNetToNeighbors(partInfo& part) { - PCU_Comm_Begin(); + int sendNetToNeighbors(partInfo& part, pcu::PCU *PCUObj) { + PCUObj->Begin(); MIS_ITERATE(vector, part.adjPartIds, adjPartIdItr) { const int destRank = *adjPartIdItr; - PCU_COMM_PACK(destRank, part.id); //FIXME redundant - PCU_COMM_PACK(destRank, *adjPartIdItr); //FIXME redundant - PCU_COMM_PACK(destRank, part.randNum); + PCUObj->Pack(destRank, part.id); //FIXME redundant + PCUObj->Pack(destRank, *adjPartIdItr); //FIXME redundant + PCUObj->Pack(destRank, part.randNum); size_t netSz = part.net.size(); - PCU_COMM_PACK(destRank, netSz); + PCUObj->Pack(destRank, netSz); MIS_ITERATE(vector, part.net, pItr) - PCU_COMM_PACK(destRank, *pItr); + PCUObj->Pack(destRank, *pItr); } return 0; } - void unpackNet(vector& msg) { + void unpackNet(vector& msg, pcu::PCU *PCUObj) { int srcPartId; - PCU_COMM_UNPACK(srcPartId); - PCU_ALWAYS_ASSERT(PCU_Comm_Sender() == srcPartId); + PCUObj->Unpack(srcPartId); + PCU_ALWAYS_ASSERT(PCUObj->Sender() == srcPartId); int destPartId; - PCU_COMM_UNPACK(destPartId); - PCU_ALWAYS_ASSERT(PCU_Comm_Self() == destPartId); + PCUObj->Unpack(destPartId); + PCU_ALWAYS_ASSERT(PCUObj->Self() == destPartId); unsigned randNum; - PCU_COMM_UNPACK(randNum); + PCUObj->Unpack(randNum); adjPart ap; ap.partId = srcPartId; ap.randNum = randNum; size_t arrayLen; - PCU_COMM_UNPACK(arrayLen); + PCUObj->Unpack(arrayLen); PCU_ALWAYS_ASSERT(arrayLen > 0); int buff; for(size_t i=0; i < arrayLen; i++) { - PCU_COMM_UNPACK(buff); + PCUObj->Unpack(buff); ap.net.push_back(buff); } msg.push_back(ap); } - void recvNetsFromNeighbors(vector& msg) { - while( PCU_Comm_Listen() ) - unpackNet(msg); + void recvNetsFromNeighbors(vector& msg, pcu::PCU *PCUObj) { + while( PCUObj->Listen() ) + unpackNet(msg, PCUObj); } - void unpackAdjPart(vector& msg) { - const int rank = PCU_Comm_Self(); + void unpackAdjPart(vector& msg, pcu::PCU *PCUObj) { + const int rank = PCUObj->Self(); //unpack destination part Id int destPartId; - PCU_COMM_UNPACK(destPartId); + PCUObj->Unpack(destPartId); PCU_ALWAYS_ASSERT(rank == destPartId); //unpack adjacent part Id int adjPartId; - PCU_COMM_UNPACK(adjPartId); + PCUObj->Unpack(adjPartId); //unpack random number unsigned randNum; - PCU_COMM_UNPACK(randNum); + PCUObj->Unpack(randNum); adjPart ap; ap.partId = adjPartId; @@ -209,21 +209,21 @@ namespace { //unpack net size size_t n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); //unpack net int pid; for(size_t i=0; i < n; i++) { - PCU_COMM_UNPACK(pid); + PCUObj->Unpack(pid); ap.net.push_back(pid); } msg.push_back(ap); } - void recvAdjNetsFromNeighbors(vector& msg) { - while( PCU_Comm_Receive() ) - unpackAdjPart(msg); + void recvAdjNetsFromNeighbors(vector& msg, pcu::PCU *PCUObj) { + while( PCUObj->Receive() ) + unpackAdjPart(msg, PCUObj); } int minRandNum(mapiuItr first, mapiuItr last) { @@ -302,25 +302,25 @@ namespace { * @param part (InOut) local part * @return 0 on success, non-zero otherwise */ - int constructNetGraph(partInfo& part) { + int constructNetGraph(partInfo& part, pcu::PCU *PCUObj) { stable_sort(part.adjPartIds.begin(), part.adjPartIds.end()); setNodeStateInGraph(part); // send net to each neighbor - int ierr = sendNetToNeighbors(part); + int ierr = sendNetToNeighbors(part, PCUObj); if (ierr != 0) return ierr; - PCU_Comm_Send(); + PCUObj->Send(); vector nbNet; - recvNetsFromNeighbors(nbNet); + recvNetsFromNeighbors(nbNet, PCUObj); // get the random num associated with adjacent nets part.addNetNeighbors(nbNet); - ierr = sendAdjNetsToNeighbors(part, nbNet); + ierr = sendAdjNetsToNeighbors(part, nbNet, PCUObj); if (ierr != 0) return ierr; - PCU_Comm_Send(); + PCUObj->Send(); vector nbAdjNets; - recvAdjNetsFromNeighbors(nbAdjNets); + recvAdjNetsFromNeighbors(nbAdjNets, PCUObj); // get the random num associated with adjacent nets part.addNetNeighbors(nbAdjNets); @@ -388,13 +388,13 @@ void misFinalize() {} * Setting isNeighbors true supports computing on the partition model graph * as opposed to the netgraph. */ -int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) { - PCU_ALWAYS_ASSERT(PCU_Comm_Initialized()); +int mis(partInfo& part, pcu::PCU *PCUObj, bool randNumsPredefined, bool isNeighbors) { + PCU_ALWAYS_ASSERT(PCUObj != nullptr); if (false == randNumsPredefined) - setRandomNum(part); + setRandomNum(part, PCUObj); - int ierr = constructNetGraph(part); + int ierr = constructNetGraph(part, PCUObj); if (ierr != 0) return ierr; vector nodesRemoved; @@ -429,9 +429,9 @@ int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) { //--------------ROUND tag++; - sendIntsToNeighbors(part, nodesToRemove, tag); - PCU_Comm_Send(); - recvIntsFromNeighbors(rmtNodesToRemove, tag); + sendIntsToNeighbors(part, nodesToRemove, tag, PCUObj); + PCUObj->Send(); + recvIntsFromNeighbors(rmtNodesToRemove, tag, PCUObj); if (true == part.isInNetGraph && @@ -451,15 +451,15 @@ int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) { //--------------ROUND tag++; - sendIntsToNeighbors(part, nodesRemoved, tag); - PCU_Comm_Send(); - recvIntsFromNeighbors(rmtNodesToRemove, tag); + sendIntsToNeighbors(part, nodesRemoved, tag, PCUObj); + PCUObj->Send(); + recvIntsFromNeighbors(rmtNodesToRemove, tag, PCUObj); removeNodes(part, rmtNodesToRemove); nodesRemoved.clear(); rmtNodesToRemove.clear(); - numNodesAdded = PCU_Add_Int(numNodesAdded); + numNodesAdded = PCUObj->Add(numNodesAdded); } while (numNodesAdded > 0); return isInMis; diff --git a/parma/diffMC/maximalIndependentSet/test/testMIS.cc b/parma/diffMC/maximalIndependentSet/test/testMIS.cc index a96c98d82..68882d35d 100644 --- a/parma/diffMC/maximalIndependentSet/test/testMIS.cc +++ b/parma/diffMC/maximalIndependentSet/test/testMIS.cc @@ -6,8 +6,8 @@ #include "math.h" #include #include -#include "PCU.h" #include "parma_commons.h" +#include using std::set; using std::vector; @@ -273,9 +273,9 @@ are removed: |MIS| = 2 */ -int test_2dStencil(const int rank, const int totNumParts, +int test_2dStencil(const int rank, const int totNumParts, pcu::PCU *PCUObj bool randNumsPredefined = false, bool isNeighbors = false) { - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_2dStencil - totNumParts: %d\n", totNumParts); const int sqrtTotNumParts = floor(sqrt(totNumParts)); @@ -299,16 +299,16 @@ int test_2dStencil(const int rank, const int totNumParts, //sanity check checkAdjPartsandNets(part, totNumParts); - const double t1 = PCU_Time(); - int isInMis = mis(part, randNumsPredefined,isNeighbors); - double elapsedTime = PCU_Time() - t1; - PCU_Max_Doubles(&elapsedTime, 1); + const double t1 = pcu::Time(); + int isInMis = mis(part, PCUObj, randNumsPredefined,isNeighbors); + double elapsedTime = pcu::Time() - t1; + PCUObj->Max(&elapsedTime, 1); - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("elapsed time (seconds) = %f \n", elapsedTime); int* globalIsInMIS = NULL; - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) globalIsInMIS = new int[totNumParts]; MPI_Gather(&isInMis, 1, MPI_INT, globalIsInMIS, 1, MPI_INT, 0, MPI_COMM_WORLD); @@ -363,10 +363,10 @@ int test_2dStencil(const int rank, const int totNumParts, * @param randNumsPredefined (In) T:rand nums predefined, F:rand nums not predefined * @return 0 on success, non-zero otherwise */ -int test_StarA(const int rank, const int totNumParts, +int test_StarA(const int rank, const int totNumParts, pcu::PCU *PCUObj, bool randNumsPredefined = false) { char dbgMsg[256]; - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_StarA - totNumParts: %d\n", totNumParts); partInfo part; @@ -414,7 +414,7 @@ int test_StarA(const int rank, const int totNumParts, Print ::iterator > (cout, dbgMsg, part.net.begin(), part.net.end(), std::string(", ")); - int isInMIS = mis(part, randNumsPredefined); + int isInMIS = mis(part, PCUObj, randNumsPredefined); int* globalIsInMIS = NULL; if (rank == 0) { @@ -466,13 +466,13 @@ randNums 5 1 13 6 */ int test_4partsA(const int rank, const int totNumParts, - bool predefinedRandNums) { + bool predefinedRandNums, pcu::PCU *PCUObj) { if (4 != totNumParts) { MIS_FAIL("totNumParts must be 4\n"); return 1; } - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_4partsA - totNumParts: %d\n", totNumParts); partInfo part; @@ -511,7 +511,7 @@ int test_4partsA(const int rank, const int totNumParts, vector parts; parts.push_back(part); - int isInMis = mis(parts[0], predefinedRandNums); + int isInMis = mis(parts[0], PCUObj, predefinedRandNums); int* globalIsInMIS = NULL; if (!rank) @@ -542,13 +542,13 @@ int test_4partsA(const int rank, const int totNumParts, } int test_4partsB(const int rank, const int totNumParts, - bool predefinedRandNums) { + bool predefinedRandNums, pcu::PCU *PCUObj) { if (4 != totNumParts) { MIS_FAIL("totNumParts must be 4\n"); return 1; } - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_4partsAndNums - totNumParts: %d\n", totNumParts); partInfo part; @@ -583,7 +583,7 @@ int test_4partsB(const int rank, const int totNumParts, vector parts; parts.push_back(part); - int isInMis = mis(parts[0], predefinedRandNums); + int isInMis = mis(parts[0], PCUObj, predefinedRandNums); int* globalIsInMIS = NULL; if (rank == 0) { @@ -615,13 +615,13 @@ int test_4partsB(const int rank, const int totNumParts, } int test_4partsC(const int rank, const int totNumParts, - bool predefinedRandNums) { + bool predefinedRandNums, pcu::PCU *PCUObj) { if (4 != totNumParts) { MIS_FAIL("totNumParts must be 4\n"); return 1; } - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_4partsA - totNumParts: %d\n", totNumParts); partInfo part; @@ -651,7 +651,7 @@ int test_4partsC(const int rank, const int totNumParts, vector parts; parts.push_back(part); - int isInMis = mis(parts[0], predefinedRandNums); + int isInMis = mis(parts[0], PCUObj, predefinedRandNums); int* globalIsInMIS = NULL; if (rank == 0) { @@ -711,12 +711,13 @@ bool getBool(int& isDebug) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); - PCU_Protect(); - int rank; - PCU_Comm_Rank(&rank); - int commSize; - PCU_Comm_Size(&commSize); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::Protect(); + int rank = PCUObj.get()->Self(); + //PCU_Comm_Rank(&rank); + int commSize = PCUObj.get()->Peers(); + //PCU_Comm_Size(&commSize); int opt; int debugMode = 0; @@ -748,7 +749,7 @@ int main(int argc, char** argv) { testNum = broadcastInt(testNum); - randNumSeed = broadcastInt(randNumSeed) + PCU_Comm_Self(); + randNumSeed = broadcastInt(randNumSeed) + PCUObj.get()->Self(); bool predefinedRandNums = getBool(iPredefinedRandNums); mis_init(randNumSeed, getBool(debugMode)); @@ -765,22 +766,22 @@ int main(int argc, char** argv) { return 0; break; case 0: - ierr = test_4partsA(rank, commSize, predefinedRandNums); + ierr = test_4partsA(rank, commSize, predefinedRandNums, PCUObj.get()); break; case 1: - ierr = test_4partsB(rank, commSize, predefinedRandNums); + ierr = test_4partsB(rank, commSize, predefinedRandNums, PCUObj.get()); break; case 2: - ierr = test_4partsC(rank, commSize, predefinedRandNums); + ierr = test_4partsC(rank, commSize, predefinedRandNums, PCUObj.get()); break; case 3: - ierr = test_StarA(rank, commSize); + ierr = test_StarA(rank, commSize, PCUObj.get()); break; case 4: - ierr = test_2dStencil(rank, commSize); + ierr = test_2dStencil(rank, commSize, PCUObj.get()); break; case 5: - ierr = test_2dStencil(rank,commSize,false,true); + ierr = test_2dStencil(rank,commSize,PCUObj.get(),false,true); break; } @@ -792,7 +793,7 @@ int main(int argc, char** argv) { } } misFinalize(); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/parma/diffMC/parma_dcpartFixer.cc b/parma/diffMC/parma_dcpartFixer.cc index d39e6c59e..6bb88f1c4 100644 --- a/parma/diffMC/parma_dcpartFixer.cc +++ b/parma/diffMC/parma_dcpartFixer.cc @@ -22,7 +22,7 @@ namespace { } } part.net.push_back(part.id); - return mis(part,false,true); + return mis(part,PCUObj,false,true); } } diff --git a/parma/parma.cc b/parma/parma.cc index 724952182..18dcba083 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -412,7 +412,7 @@ int Parma_MisNumbering(apf::Mesh* m, int d) { int iter=0; int misSize=0; while( misSize != m->getPCU()->Peers() ) { - if( mis(part, false, true) || 1 == part.net.size() ) { + if( mis(part, m->getPCU(), false, true) || 1 == part.net.size() ) { misNumber = iter; part.net.clear(); part.adjPartIds.clear(); diff --git a/parma/parma.h b/parma/parma.h index 065477fa7..3d65c9779 100644 --- a/parma/parma.h +++ b/parma/parma.h @@ -299,7 +299,7 @@ struct Parma_GroupCode * the group. * @param group the group id number, starting from zero */ - virtual void run(int group) = 0; + virtual void run(int group, pcu::PCU *PCUObj) = 0; }; /** @@ -327,7 +327,7 @@ void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); * apf::remapPartition is used to maintain the mesh structure * during these transitions. */ -void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); +void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj); /** * @brief Compute maximal independent set numbering From 8cb06bba9b301f998fea043dda402eb511b75524 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 9 Apr 2024 15:02:50 -0400 Subject: [PATCH 100/141] Parma_GroupCode run() function default changed to take a PCUObj --- phasta/condense.cc | 2 +- phasta/phAdapt.cc | 4 ++-- phasta/phCook.cc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/phasta/condense.cc b/phasta/condense.cc index 21e5e95d5..f7d9e2fc4 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -21,7 +21,7 @@ namespace { struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; ph::Input ctrl; - void run(int) { + void run(int, pcu::PCU*) { apf::reorderMdsMesh(mesh,NULL); chef::preprocess(mesh,ctrl); } diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index a40ada8c4..29e99256c 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -55,7 +55,7 @@ struct AdaptCallback : public Parma_GroupCode : mesh(m), field(szfld), in(NULL) { } AdaptCallback(apf::Mesh2* m, apf::Field* szfld, ph::Input* inp) : mesh(m), field(szfld), in(inp) { } - void run(int) { + void run(int, pcu::PCU*) { ma::Input* ma_in = ma::makeAdvanced(ma::configure(mesh, field)); if( in ) { //chef defaults @@ -125,7 +125,7 @@ void adaptShrunken(apf::Mesh2* m, double minPartDensity, if (!m->getPCU()->Self()) lion_eprint(1,"adaptShrunken limit set to %f factor computed as %d\n", minPartDensity, factor); if (factor == 1) { - callback.run(0); + callback.run(0, m->getPCU()); } else { warnAboutShrinking(factor, m->getPCU()); Parma_ShrinkPartition(m, factor, callback); diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 7dfbfdb8b..f4db9f7be 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -248,7 +248,7 @@ namespace { ph::Input* input; ph::BCs* boundary; apf::Mesh2* mesh; - void run(int) { + void run(int, pcu::PCU*) { ph::Output groupOut; //streaming not supported from group code! groupOut.openfile_write = chef::openfile_write; From f36021a5b532e00a14cb170902f756f62af339e5 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 9 Apr 2024 15:04:31 -0400 Subject: [PATCH 101/141] Test files fixed --- test/bezierMesh.cc | 45 ++++++++++++++++--------------- test/bezierMisc.cc | 15 ++++++----- test/bezierRefine.cc | 29 ++++++++++---------- test/bezierShapeEval.cc | 23 ++++++++-------- test/bezierSubdivision.cc | 39 ++++++++++++++------------- test/bezierValidity.cc | 27 ++++++++++--------- test/capVol.cc | 10 +++---- test/cgns.cc | 26 ++++++++++-------- test/classifyThenAdapt.cc | 13 ++++----- test/collapse.cc | 5 ++-- test/construct.cc | 11 ++++---- test/constructThenGhost.cc | 1 - test/construct_bottom_up.cc | 12 ++++----- test/convert.cc | 29 ++++++++++---------- test/crack_test.cc | 22 ++++++++------- test/create_mis.cc | 11 ++++---- test/curve_to_bezier.cc | 13 ++++----- test/curvetest.cc | 31 +++++++++++----------- test/degenerateSurfs.cc | 15 ++++++----- test/describe.cc | 41 ++++++++++++++-------------- test/dg_ma_test.cc | 9 ++++--- test/elmBalance.cc | 11 ++++---- test/embedded_edges.cc | 11 ++++---- test/extrude.cc | 11 ++++---- test/fieldReduce.cc | 23 +++++++++------- test/field_io.cc | 11 ++++---- test/fixDisconnected.cc | 11 ++++---- test/fixlayer.cc | 9 ++++--- test/fixshape.cc | 9 ++++--- test/fusion.cc | 17 ++++++------ test/fusion2.cc | 21 ++++++++------- test/fusion3.cc | 10 ++++--- test/gap.cc | 11 ++++---- test/generate.cc | 53 +++++++++++++++++++------------------ test/poisson.cc | 2 +- test/repartition.cc | 3 ++- test/serialize.cc | 13 ++++----- test/spr_test.cc | 11 ++++---- 38 files changed, 352 insertions(+), 312 deletions(-) diff --git a/test/bezierMesh.cc b/test/bezierMesh.cc index bc6995a77..da106938d 100644 --- a/test/bezierMesh.cc +++ b/test/bezierMesh.cc @@ -5,11 +5,11 @@ #include #include #include -#include #include #include #include #include +#include /* this test file contains tests for * a curved 2D mesh @@ -121,10 +121,10 @@ gmi_model* makeModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[4]; apf::Vector3 points2D[4] = {apf::Vector3(0,0,0), @@ -153,10 +153,10 @@ apf::Mesh2* createMesh2D() return m; } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::Vector3 points3D[4] = {apf::Vector3(0,0,0), @@ -253,12 +253,12 @@ void testSize2D(apf::Mesh2* m, int order) } } -void test2D() +void test2D(pcu::PCU *PCUObj) { // test all orders for all blending orders for(int order = 1; order <= 6; ++order){ for(int blendOrder = 0; blendOrder <= 2; ++blendOrder){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,order,blendOrder); bc.run(); testInterpolatedPoints2D(m); @@ -269,7 +269,7 @@ void test2D() } // test the pure interpolation side of things { - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); apf::changeMeshShape(m,apf::getLagrange(2),true); crv::InterpolatingCurver ic(m,2); ic.run(); @@ -467,10 +467,10 @@ void test3D(apf::Mesh2* m) * triangles implemented. There are no nodes inside the tetrahedron * */ -void test3DBlended() +void test3DBlended(pcu::PCU *PCUObj) { gmi_register_null(); - apf::Mesh2* mbase = createMesh3D(); + apf::Mesh2* mbase = createMesh3D(PCUObj); testSize3D(mbase); test3DJacobian(mbase); @@ -481,7 +481,7 @@ void test3DBlended() for(int order = 1; order <= 6; ++order){ for(int blendOrder = 1; blendOrder <= 2; ++blendOrder){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,blendOrder); bc.run(); test3D(m); @@ -492,12 +492,12 @@ void test3DBlended() } /* Tests Full Bezier tetrahedra */ -void test3DFull() +void test3DFull(pcu::PCU *PCUObj) { gmi_register_null(); for(int order = 1; order <= 6; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); test3D(m); @@ -540,7 +540,7 @@ void test3DFull() } // test simple elevation for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); crv::changeMeshOrder(m,5); @@ -550,7 +550,7 @@ void test3DFull() } // test elevation inside a BezierCurver for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc1(m,order,0); bc1.run(); crv::BezierCurver bc2(m,order+2,0); @@ -561,7 +561,7 @@ void test3DFull() } // test going downward for(int order = 4; order <= 6; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc1(m,order,0); bc1.run(); crv::BezierCurver bc2(m,order-2,0); @@ -572,7 +572,7 @@ void test3DFull() } // test going from 2nd order lagrange to various order bezier for(int order = 2; order <= 6; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); apf::changeMeshShape(m,apf::getLagrange(2),true); crv::BezierCurver bc(m,order,0); bc.run(); @@ -585,11 +585,12 @@ void test3DFull() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - test2D(); - test3DBlended(); - test3DFull(); - PCU_Comm_Free(); + test2D(pcu_obj.get()); + test3DBlended(pcu_obj.get()); + test3DFull(pcu_obj.get()); + } MPI_Finalize(); } diff --git a/test/bezierMisc.cc b/test/bezierMisc.cc index 77c4b0d95..4af003d20 100644 --- a/test/bezierMisc.cc +++ b/test/bezierMisc.cc @@ -9,12 +9,12 @@ #include #include #include -#include #include #include #include #include #include +#include /* This file contains miscellaneous tests relating to ordering, math * and transformation matrices */ @@ -25,10 +25,10 @@ static apf::Vector3 points3D[4] = apf::Vector3(0,1,0), apf::Vector3(0,0,1)}; -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TRIANGLE,points3D); apf::deriveMdsModel(m); @@ -38,10 +38,10 @@ apf::Mesh2* createMesh2D() return m; } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TET,points3D); apf::deriveMdsModel(m); @@ -167,11 +167,12 @@ void testMatrixInverse(){ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); testNodeIndexing(); testMatrixInverse(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/bezierRefine.cc b/test/bezierRefine.cc index 681930071..be9a8d81a 100644 --- a/test/bezierRefine.cc +++ b/test/bezierRefine.cc @@ -9,11 +9,11 @@ #include #include #include -#include #include #include #include +#include // face areas are 1/2 and 19/30 void vert0(double const p[2], double x[3], void*) @@ -138,10 +138,10 @@ gmi_model* makeModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[4]; apf::Vector3 points2D[4] = {apf::Vector3(0,0,0), @@ -183,10 +183,10 @@ static double measureMesh(apf::Mesh2* m) return v; } -void test2D() +void test2D(pcu::PCU *PCUObj) { for(int order = 1; order <= 6; ++order){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); apf::changeMeshShape(m, crv::getBezier(order),true); crv::BezierCurver bc(m,order,0); if(order > 1){ @@ -239,10 +239,10 @@ void test2D() } } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); double x = 1./sqrt(6.); double z = 1./sqrt(12.); apf::Vector3 points3D[4] = @@ -260,12 +260,12 @@ apf::Mesh2* createMesh3D() return m; } -void test3D() +void test3D(pcu::PCU *PCUObj) { gmi_register_null(); // test full for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -285,7 +285,7 @@ void test3D() } // test blended for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,1); bc.run(); @@ -311,10 +311,11 @@ void test3D() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - test2D(); - test3D(); - PCU_Comm_Free(); + test2D(pcu_obj.get()); + test3D(pcu_obj.get()); + } MPI_Finalize(); } diff --git a/test/bezierShapeEval.cc b/test/bezierShapeEval.cc index 07abf4785..db78f5f5f 100644 --- a/test/bezierShapeEval.cc +++ b/test/bezierShapeEval.cc @@ -10,27 +10,28 @@ #include #include #include -#include #include #include #include #include #include +#include /* This file contains miscellaneous tests relating to bezier shapes and * blended bezier shapes */ -static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent); -static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent); +static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj); +static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 7 ) { - if ( !PCU_Comm_Self() ) { + if ( !PCUObj.get()->Self() ) { printf("Usage: %s >\n", argv[0]); printf(" can only be 2 (for triangles) and 4 (for tets)\n"); printf(" is the order of bezier\n"); @@ -50,7 +51,7 @@ int main(int argc, char** argv) " must be between -1 and 2!"); apf::MeshEntity* ent = 0; - apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent) : makeOneTetMesh(order,ent); + apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent,PCUObj.get()) : makeOneTetMesh(order,ent,PCUObj.get()); double xi0 = atof(argv[4]); double xi1 = atof(argv[5]); @@ -80,13 +81,13 @@ int main(int argc, char** argv) printf("%d, %E \n", i, vals[i]); } - PCU_Comm_Free(); + } MPI_Finalize(); } -static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent) +static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj) { - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 2, false, PCUObj); double vert_coords[3][6] = { {0.,0.,0., 0., 0., 0.}, @@ -142,10 +143,10 @@ static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent) return mesh; } -static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent) +static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj) { - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 3, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 3, false, PCUObj); double vert_coords[4][6] = { {0.,0.,0., 0., 0., 0.}, diff --git a/test/bezierSubdivision.cc b/test/bezierSubdivision.cc index 9eb99d329..111f65f45 100644 --- a/test/bezierSubdivision.cc +++ b/test/bezierSubdivision.cc @@ -6,11 +6,11 @@ #include #include #include -#include #include #include #include #include +#include /* * This contains all the tests for bezier subdivision @@ -131,10 +131,10 @@ gmi_model* makeFaceModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeFaceModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[3], *edges[3]; apf::Vector3 points2D[3] = {apf::Vector3(0,0,0),apf::Vector3(1,0,0),apf::Vector3(1,1,0)}; @@ -162,10 +162,10 @@ static apf::Vector3 points3D[4] = apf::Vector3(0,0,1)}; -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TET,points3D); apf::deriveMdsModel(m); @@ -182,12 +182,12 @@ apf::Mesh2* createMesh3D() * to compare correctness of the split. * */ -void testEdgeSubdivision() +void testEdgeSubdivision(pcu::PCU *PCUObj) { for (int o = 1; o <= 6; ++o){ gmi_model* model = makeEdgeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false, PCUObj); apf::ModelEntity* edgeModel = m->findModelEntity(1,0); @@ -259,11 +259,11 @@ void testEdgeSubdivision() * exactly replicates its part of the old one * */ -void testTriSubdivision1() +void testTriSubdivision1(pcu::PCU *PCUObj) { for(int o = 1; o <= 6; ++o){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,o,0); bc.run(); @@ -347,10 +347,10 @@ void testTriSubdivision1() /* Create a single triangle, split it into 4, and try not to crash * */ -void testTriSubdivision4() +void testTriSubdivision4(pcu::PCU *PCUObj) { for(int o = 2; o <= 2; ++o){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,o,0); bc.run(); apf::MeshIterator* it = m->begin(2); @@ -379,14 +379,14 @@ void testTriSubdivision4() * exactly replicates its part of the old one * */ -void testTetSubdivision1() +void testTetSubdivision1(pcu::PCU *PCUObj) { gmi_register_null(); for (int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -482,12 +482,13 @@ void testTetSubdivision1() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - testEdgeSubdivision(); - testTriSubdivision1(); - testTriSubdivision4(); - testTetSubdivision1(); - PCU_Comm_Free(); + testEdgeSubdivision(pcu_obj.get()); + testTriSubdivision1(pcu_obj.get()); + testTriSubdivision4(pcu_obj.get()); + testTetSubdivision1(pcu_obj.get()); + } MPI_Finalize(); } diff --git a/test/bezierValidity.cc b/test/bezierValidity.cc index 694292114..b0bf0657a 100644 --- a/test/bezierValidity.cc +++ b/test/bezierValidity.cc @@ -9,11 +9,11 @@ #include #include #include -#include #include #include #include +#include /* This test file uses an alternative and more traditional method to * compute Jacobian differences, using the property of Bezier's that @@ -186,10 +186,10 @@ gmi_model* makeModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[4]; apf::Vector3 points2D[4] = {apf::Vector3(0,0,0), @@ -245,10 +245,10 @@ void checkValidity(apf::Mesh* m, int order) m->end(it); } -void test2D() +void test2D(pcu::PCU *PCUObj) { for(int order = 2; order <= 6; ++order){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); apf::changeMeshShape(m, crv::getBezier(order),true); crv::BezierCurver bc(m,order,0); // creates interpolation points based on the edges of the geometry @@ -299,10 +299,10 @@ void test2D() } } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::Vector3 points3D[4] = {apf::Vector3(0,0,0), @@ -319,12 +319,12 @@ apf::Mesh2* createMesh3D() return m; } -void test3D() +void test3D(pcu::PCU *PCUObj) { gmi_register_null(); for(int order = 2; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); apf::changeMeshShape(m, crv::getBezier(order),true); apf::FieldShape* fs = m->getShape(); crv::BezierCurver bc(m,order,0); @@ -407,10 +407,11 @@ void test3D() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - test2D(); - test3D(); - PCU_Comm_Free(); + test2D(pcu_obj.get()); + test3D(pcu_obj.get()); + } MPI_Finalize(); } diff --git a/test/capVol.cc b/test/capVol.cc index a9149c585..87dedd34b 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -1,11 +1,11 @@ #include #include +#include // Output #include // Parallelism -#include #include // Mesh interfaces @@ -30,7 +30,6 @@ namespace { void myExit(int exit_code = EXIT_SUCCESS) { gmi_cap_stop(); - PCU_Comm_Free(); MPI_Finalize(); exit(exit_code); } @@ -77,7 +76,8 @@ void printUsage(char *argv0) { int main(int argc, char** argv) { // Initialize parallelism. MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); // Initialize logging. lion_set_stdout(stdout); @@ -85,7 +85,7 @@ int main(int argc, char** argv) { // Check arguments or print usage. if (argc < 3) { - if (PCU_Comm_Self() == 0) { + if (pcu_obj.get()->Self() == 0) { printUsage(argv[0]); } myExit(EXIT_FAILURE); @@ -271,7 +271,7 @@ int main(int argc, char** argv) { // Exit calls. gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/cgns.cc b/test/cgns.cc index 892d5e515..b5ac692af 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include // @@ -16,6 +16,7 @@ #include #include #include +#include // https://gist.github.com/bgranzow/98087114166956646da684ed98acab02 apf::MeshTag *create_int_tag(const std::string &name, apf::Mesh2 *m, int dim) @@ -114,7 +115,7 @@ pMesh toPumi(const std::string &prefix, gmi_model *g, apf::Mesh2 *mesh) pMesh pm = pumi_mesh_load(mesh); std::cout << pm << std::endl; pumi_mesh_verify(pm); - const std::string name = prefix + "_toPUMI_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_toPUMI_" + std::to_string(pm->getPCU()->Peers()) + "procs"; pumi_mesh_write(pm, name.c_str(), "vtk"); return pm; } @@ -172,12 +173,14 @@ auto additional(const std::string &prefix, gmi_model *g, apf::Mesh2 *mesh) return clean; } -std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std::string &argv2, const std::string &post, const bool &additionalTests, const bool &writeCGNS, const std::vector> &meshData) +std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std::string &argv2, const std::string &post, const bool &additionalTests, const bool &writeCGNS, const std::vector> &meshData, pcu::PCU *PCUObj) { gmi_register_null(); gmi_register_mesh(); gmi_model *g = gmi_load(".null"); - apf::Mesh2 *m = apf::loadMdsFromCGNS(g, argv1.c_str(), cgnsBCMap, meshData); + PCUHandle h; + h.ptr = static_cast(PCUObj); + apf::Mesh2 *m = apf::loadMdsFromCGNS2(h, g, argv1.c_str(), cgnsBCMap, meshData); m->verify(); // m->writeNative(argv2.c_str()); @@ -477,12 +480,13 @@ int main(int argc, char **argv) { #ifdef HAVE_CGNS MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); bool additionalTests = false; if (argc < 3) { - if (!PCU_Comm_Self()) + if (!PCUObj.get()->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -494,7 +498,7 @@ int main(int argc, char **argv) additionalTests = true; else { - if (!PCU_Comm_Self()) + if (!PCUObj.get()->Self()) printf("Usage: %s additional\n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -503,7 +507,7 @@ int main(int argc, char **argv) } else if (argc > 4) { - if (!PCU_Comm_Self()) + if (!PCUObj.get()->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -515,7 +519,7 @@ int main(int argc, char **argv) { apf::CGNSBCMap cgnsBCMap; std::vector> meshData; - cgnsOutputName = doit(cgnsBCMap, argv[1], argv[2], "", additionalTests, additionalTests, meshData); + cgnsOutputName = doit(cgnsBCMap, argv[1], argv[2], "", additionalTests, additionalTests, meshData, PCUObj.get()); } // Phase 2 if (additionalTests) @@ -538,10 +542,10 @@ int main(int argc, char **argv) meshData.push_back(std::make_pair("CellCenter", "DummyMatrix_1")); apf::CGNSBCMap cgnsBCMap; std::cout << "RE-READING " << std::endl; - doit(cgnsBCMap, cgnsOutputName.c_str(), "tempy.smb", "_reread", false, true, meshData); + doit(cgnsBCMap, cgnsOutputName.c_str(), "tempy.smb", "_reread", false, true, meshData, PCUObj.get()); } // - PCU_Comm_Free(); + } MPI_Finalize(); return 0; #else diff --git a/test/classifyThenAdapt.cc b/test/classifyThenAdapt.cc index a2d01f3d0..ad8cce792 100644 --- a/test/classifyThenAdapt.cc +++ b/test/classifyThenAdapt.cc @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #define LEFT_EDGE 0 #define TOP_EDGE 1 @@ -45,9 +45,9 @@ void createVtx(apf::Mesh2* m, int mdlDim, int mdlId, apf::Vector3 pt, EntVec& ve verts.push_back(vert); } -apf::Mesh2* createTriMesh() { +apf::Mesh2* createTriMesh(pcu::PCU *PCUObj) { gmi_model* g = gmi_load(".null"); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false, PCUObj); apf::Vector3 p0( 0.0, 1.0, 0.0); apf::Vector3 p1( 0.5, 1.0, 0.0); @@ -127,11 +127,12 @@ void printClassCounts(apf::Mesh* m) { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); - ma::Mesh* m = createTriMesh(); + ma::Mesh* m = createTriMesh(pcu_obj.get()); m->verify(); // make sure the mesh is valid! gmi_model* g = m->getModel(); gmi_write_dmg(g,"model.dmg"); @@ -148,6 +149,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/collapse.cc b/test/collapse.cc index eac060ec4..1eb05333a 100644 --- a/test/collapse.cc +++ b/test/collapse.cc @@ -2,8 +2,7 @@ #include #include #include -//#include -#include +//#include #include #ifdef HAVE_SIMMETRIX #include @@ -23,7 +22,7 @@ namespace { struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; - void run(int) { + void run(int, pcu::PCU*) { mesh->writeNative(outFile); } }; diff --git a/test/construct.cc b/test/construct.cc index a702b973d..647994c5e 100644 --- a/test/construct.cc +++ b/test/construct.cc @@ -4,15 +4,16 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -22,7 +23,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -30,7 +31,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false); + m = apf::makeEmptyMdsMesh(model, dim, false, PCUObj.get()); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; @@ -45,7 +46,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index dad66cb7e..fcf8b908b 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include diff --git a/test/construct_bottom_up.cc b/test/construct_bottom_up.cc index 2856ee2c2..4daa384cd 100644 --- a/test/construct_bottom_up.cc +++ b/test/construct_bottom_up.cc @@ -4,9 +4,9 @@ #include #include #include -#include #include #include +#include /* This executable demos how to make a PUMI mesh using a bottom up approach. * The info needed to achieve this: @@ -139,10 +139,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT_VERBOSE(PCU_Comm_Peers() == 1, "Not implemented in parallel!"); + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.get()->Peers() == 1, "Not implemented in parallel!"); if (argc < 2) { printf("USAGE 1 ( no model): %s \n", argv[0]); printf("USAGE 2 (with model): %s destroyNative(); apf::destroyMesh(outMesh); - - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/convert.cc b/test/convert.cc index 946ed7feb..50c91dc8a 100644 --- a/test/convert.cc +++ b/test/convert.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -23,6 +22,7 @@ #include #include #include +#include using namespace std; @@ -82,7 +82,7 @@ const char* extruRootPath = NULL; int ExtruRootId =0; bool found_bad_arg = false; -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { opterr = 0; @@ -120,18 +120,18 @@ void getConfig(int argc, char** argv) { gmi_native_path = optarg; break; case '?': - if (!PCU_Comm_Self()) + if (!pcu_obj.get()->Self()) printf ("warning: skipping unrecognized option \'%s\'\n", argv[optind-1]); break; default: - if (!PCU_Comm_Self()) + if (!pcu_obj.get()->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!PCU_Comm_Self()) + if (!pcu_obj.get()->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -139,7 +139,7 @@ void getConfig(int argc, char** argv) { gmi_path = argv[i++]; sms_path = argv[i++]; smb_path = argv[i++]; - if (!PCU_Comm_Self()) { + if (!pcu_obj.get()->Self()) { printf ("fix_pyramids %d attach_order %d enable_log %d extruRootPath %s\n", should_fix_pyramids, should_attach_order, should_log, extruRootPath); printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", @@ -427,7 +427,8 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); //for fancy BL/extrusion queries @@ -435,12 +436,12 @@ int main(int argc, char** argv) Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv); + getConfig(argc, argv, pcu_obj.get()); if( should_log ) Sim_logOn("convert.sim.log"); if (should_attach_order && should_fix_pyramids) { - if (!PCU_Comm_Self()) + if (!pcu_obj.get()->Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = false; } @@ -452,7 +453,7 @@ int main(int argc, char** argv) gmi_model* mdl; if( gmi_native_path ) { - if (!PCU_Comm_Self()) + if (!pcu_obj.get()->Self()) fprintf(stderr, "loading native model %s\n", gmi_native_path); mdl = gmi_sim_load(gmi_native_path,gmi_path); } else { @@ -481,15 +482,15 @@ int main(int argc, char** argv) double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); double t1 = pcu::Time(); - if(!PCU_Comm_Self()) + if(!pcu_obj.get()->Self()) fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, pcu_obj.get()); addFathersTag(simModel, sim_mesh, simApfMesh, extruRootPath); double t2 = pcu::Time(); - if(!PCU_Comm_Self()) + if(!simApfMesh->getPCU()->Self()) fprintf(stderr, "created the apf_sim mesh in %f seconds\n", t2-t1); if (should_attach_order) attachOrder(simApfMesh); @@ -517,6 +518,6 @@ int main(int argc, char** argv) MS_exit(); if( should_log ) Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/crack_test.cc b/test/crack_test.cc index 879770c0c..76f77009e 100644 --- a/test/crack_test.cc +++ b/test/crack_test.cc @@ -13,20 +13,20 @@ #include #include #include -#include #include #include #include #include #include #include +#include void bCurver(const char* modelFile, const char* meshFile, - int order, const char* outputPrefix, + pcu::PCU *PCUObj, int order, const char* outputPrefix, int blendOrder = 0) { - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); m->verify(); if (m->getPCU()->Self() == 0) printf("attempting to curve the mesh to %d order Bezier\n", order); @@ -45,17 +45,19 @@ void bCurver(const char* modelFile, const char* meshFile, int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + if (argc < 2) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); exit(EXIT_FAILURE); } - const char* modelFile = argv[1]; - MPI_Init(&argc, &argv); - PCU_Comm_Init(); + lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -68,7 +70,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_model* g = gmi_load(modelFile); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false, PCUObj.get()); std::vector verts; // vertex coordinates @@ -348,7 +350,7 @@ int main(int argc, char** argv) for (int i = 2; i < 7; i++) { char output[256]; sprintf(output,"crack_curved_to_order_%d", i); - bCurver(modelFile, "crack_linear.smb", i, output); + bCurver(modelFile, "crack_linear.smb", PCUObj.get(), i, output); } #ifdef HAVE_SIMMETRIX @@ -357,6 +359,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/create_mis.cc b/test/create_mis.cc index 40a43ddd6..50378c4d6 100644 --- a/test/create_mis.cc +++ b/test/create_mis.cc @@ -3,18 +3,19 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -22,7 +23,7 @@ int main(int argc, char** argv) gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); //We will create a field with the coloring as the values on each element apf::Field* coloring = apf::createField(m,"colors",apf::SCALAR, @@ -60,7 +61,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/curve_to_bezier.cc b/test/curve_to_bezier.cc index 70f1f2576..03ffce574 100644 --- a/test/curve_to_bezier.cc +++ b/test/curve_to_bezier.cc @@ -5,31 +5,32 @@ #include #include #include -#include #include #include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } int order = atoi(argv[3]); if(order < 1 || order > 6){ - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Only 1st to 6th order supported\n"); MPI_Finalize(); exit(EXIT_FAILURE); @@ -37,7 +38,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_register_sim(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); crv::BezierCurver bc(m,order,0); bc.run(); @@ -53,6 +54,6 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/curvetest.cc b/test/curvetest.cc index 1c1f7585d..90369224e 100644 --- a/test/curvetest.cc +++ b/test/curvetest.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -12,6 +11,7 @@ #include #include #include +#include class Linear : public ma::IsotropicFunction { @@ -85,10 +85,10 @@ static void testElementSize(apf::Mesh* m) } static void testInterpolating(const char* modelFile, const char* meshFile, - const int ne, const int nf) + const int ne, const int nf, pcu::PCU *PCUObj) { for(int order = 1; order <= 2; ++order){ - apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile,PCUObj); apf::changeMeshShape(m2,apf::getLagrange(order),true); crv::InterpolatingCurver ic(m2,order); ic.run(); @@ -103,7 +103,7 @@ static void testInterpolating(const char* modelFile, const char* meshFile, } static void testBezier(const char* modelFile, const char* meshFile, - const int ne, const int nf) + const int ne, const int nf, pcu::PCU *PCUObj) { apf::DynamicMatrix edgeErrors(ne,6); @@ -111,7 +111,7 @@ static void testBezier(const char* modelFile, const char* meshFile, // check blended shapes + interpolation error // interpolation error decreases as order is increased for(int order = 1; order <= 6; ++order){ - apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::BezierCurver bc(m2,order,2); bc.run(); testElementSize(m2); @@ -138,7 +138,7 @@ static void testBezier(const char* modelFile, const char* meshFile, } // check some refinement for(int order = 2; order <= 4; ++order){ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); Linear sf(m); @@ -153,7 +153,7 @@ static void testBezier(const char* modelFile, const char* meshFile, } // check some refinement for blended shapes for(int order = 2; order <= 4; ++order){ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::BezierCurver bc(m,order,1); bc.run(); Linear sf(m); @@ -170,10 +170,10 @@ static void testBezier(const char* modelFile, const char* meshFile, } static void testGregory(const char* modelFile, const char* meshFile, - const int ne, const int nf) + const int ne, const int nf, pcu::PCU *PCUObj) { for(int order = 0; order <= 2; ++order){ - apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::GregoryCurver gc(m2,4,order); gc.run(); testElementSize(m2); @@ -192,24 +192,25 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); int ne = m->count(1); int nf = m->count(2); m->destroyNative(); apf::destroyMesh(m); - testInterpolating(modelFile,meshFile,ne,nf); - testBezier(modelFile,meshFile,ne,nf); - testGregory(modelFile,meshFile,ne,nf); + testInterpolating(modelFile,meshFile,ne,nf,PCUObj.get()); + testBezier(modelFile,meshFile,ne,nf,PCUObj.get()); + testGregory(modelFile,meshFile,ne,nf,PCUObj.get()); - PCU_Comm_Free(); + } gmi_sim_stop(); Sim_unregisterAllKeys(); SimModel_stop(); diff --git a/test/degenerateSurfs.cc b/test/degenerateSurfs.cc index 730e8cbf7..91dd9f94b 100644 --- a/test/degenerateSurfs.cc +++ b/test/degenerateSurfs.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,16 +12,17 @@ #endif #include #include +#include const char* modelFile = 0; const char* meshFile = 0; const char* outFile = 0; int level = 1; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -37,7 +37,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==5); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -47,8 +48,8 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); int order = 2; @@ -71,7 +72,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/describe.cc b/test/describe.cc index c7171f287..bbd149cc0 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -17,11 +16,12 @@ #include #endif #include +#include #ifdef __bgq__ #include -static double get_peak() +static double get_peak(pcu::PCU*) { uint64_t heap; Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap); @@ -30,7 +30,7 @@ static double get_peak() #elif defined (__linux__) -static double get_peak() +static double get_peak(pcu::PCU*) { #if defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2) return mallinfo2().arena; @@ -41,30 +41,30 @@ static double get_peak() #else -static double get_peak() +static double get_peak(pcu::PCU *PCUObj) { - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("%s:%d: OS Not supported\n", __FILE__, __LINE__); return(-1.0); } #endif -static void print_stats(const char* name, double value) +static void print_stats(const char* name, double value, pcu::PCU *PCUObj) { double min, max, avg; - min = PCU_Min_Double(value); - max = PCU_Max_Double(value); - avg = PCU_Add_Double(value); - avg /= PCU_Comm_Peers(); + min = PCUObj->Min(value); + max = PCUObj->Max(value); + avg = PCUObj->Add(value); + avg /= PCUObj->Peers(); double imb = max / avg; - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb); } #if defined(__linux__) -static double get_chunks() +static double get_chunks(pcu::PCU*) { #if defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2) struct mallinfo2 m = mallinfo2(); @@ -75,9 +75,9 @@ static double get_chunks() } #else -static double get_chunks() +static double get_chunks(pcu::PCU *PCUObj) { - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("%s:%d: OS Not supported\n", __FILE__, __LINE__); return(-1.0); } @@ -97,7 +97,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -107,11 +108,11 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - print_stats("malloc used before", get_chunks()); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + print_stats("malloc used before", get_chunks(pcu_obj.get()), pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); m->verify(); - print_stats("kernel heap", get_peak()); - print_stats("malloc used", get_chunks()); + print_stats("kernel heap", get_peak(pcu_obj.get()), pcu_obj.get()); + print_stats("malloc used", get_chunks(pcu_obj.get()), pcu_obj.get()); Parma_PrintPtnStats(m, ""); list_tags(m); m->destroyNative(); @@ -122,6 +123,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/dg_ma_test.cc b/test/dg_ma_test.cc index c29ff5ce1..9ddec672d 100644 --- a/test/dg_ma_test.cc +++ b/test/dg_ma_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -12,6 +11,7 @@ #include #endif #include +#include class Linear : public ma::IsotropicFunction { @@ -41,7 +41,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -51,7 +52,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); Linear sf(m); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); @@ -76,7 +77,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/elmBalance.cc b/test/elmBalance.cc index 5ce81d5d7..ab5e806c7 100644 --- a/test/elmBalance.cc +++ b/test/elmBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #endif #include #include +#include apf::MeshTag* setWeights(apf::Mesh* m) { apf::MeshIterator* it = m->begin(m->getDimension()); @@ -29,10 +29,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -45,7 +46,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); if(!m->getPCU()->Self()) @@ -67,6 +68,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/embedded_edges.cc b/test/embedded_edges.cc index be0948569..73f07230f 100644 --- a/test/embedded_edges.cc +++ b/test/embedded_edges.cc @@ -4,11 +4,11 @@ #include #include #include -#include #include #include #include #include +#include // this test checks that the destruct function works // with meshes that have a lower dimension than the manifold // which tey reside in. E.g. a truss or beam in 3D space @@ -17,7 +17,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -28,7 +29,7 @@ int main(int argc, char** argv) int nverts; gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsMesh(model, argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(model, argv[1], PCUObj.get()); apf::deriveMdsModel(m); int dim = m->getDimension(); extractCoords(m, coords, nverts); @@ -40,7 +41,7 @@ int main(int argc, char** argv) std::cout<typeDimension[apf::Mesh::EDGE]<destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/extrude.cc b/test/extrude.cc index f3c5fc1d6..a689b729a 100644 --- a/test/extrude.cc +++ b/test/extrude.cc @@ -2,25 +2,26 @@ #include #include #include -#include #include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); ma::ModelExtrusions extrusions; extrusions.push_back(ma::ModelExtrusion( m->findModelEntity(1, 2), @@ -39,6 +40,6 @@ int main(int argc, char** argv) m->writeNative(argv[4]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fieldReduce.cc b/test/fieldReduce.cc index b33fea701..aad2a1127 100644 --- a/test/fieldReduce.cc +++ b/test/fieldReduce.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -16,6 +15,7 @@ #include #include #include +#include namespace { @@ -147,10 +147,10 @@ void freeMesh(apf::Mesh* m) apf::destroyMesh(m); } -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -166,10 +166,12 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + bool failflag = false; + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - MPI_Comm_rank(PCU_Get_Comm(), &myrank); - MPI_Comm_size(PCU_Get_Comm(), &commsize); + MPI_Comm_rank(PCUObj.get()->GetMPIComm(), &myrank); + MPI_Comm_size(PCUObj.get()->GetMPIComm(), &commsize); #ifdef HAVE_SIMMETRIX MS_init(); SimModel_start(); @@ -178,13 +180,13 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,PCUObj.get()); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; - m = apf::loadMdsMesh(g, meshFile); + m = apf::loadMdsMesh(g, meshFile, PCUObj.get()); - bool failflag = false; + for (int i=0; i < 3; ++i) failflag = failflag || testReduce(m, i); @@ -195,9 +197,10 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); return failflag; + } diff --git a/test/field_io.cc b/test/field_io.cc index d1514c1db..36f39be10 100644 --- a/test/field_io.cc +++ b/test/field_io.cc @@ -3,19 +3,20 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); { - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); apf::Field* f = apf::createLagrangeField(m, "foo", apf::VECTOR, 1); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* vert; @@ -28,7 +29,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); } { - apf::Mesh2* m = apf::loadMdsMesh(argv[1], "tmp.smb"); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], "tmp.smb", PCUObj.get()); apf::Field* f = m->findField("foo"); PCU_ALWAYS_ASSERT(f); PCU_ALWAYS_ASSERT(apf::VECTOR == apf::getValueType(f)); @@ -46,6 +47,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fixDisconnected.cc b/test/fixDisconnected.cc index 5d7174c49..492755efa 100644 --- a/test/fixDisconnected.cc +++ b/test/fixDisconnected.cc @@ -3,31 +3,32 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); Parma_PrintPtnStats(m, "initial"); Parma_ProcessDisconnectedParts(m); Parma_PrintPtnStats(m, "final"); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fixlayer.cc b/test/fixlayer.cc index 0827bc025..dd5fd26d9 100644 --- a/test/fixlayer.cc +++ b/test/fixlayer.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -11,12 +10,14 @@ #include #endif #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +27,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldCleanupLayer = true; ma::adapt(in); @@ -39,7 +40,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fixshape.cc b/test/fixshape.cc index 899e1c0e4..048bc0d61 100644 --- a/test/fixshape.cc +++ b/test/fixshape.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -11,12 +10,14 @@ #include #endif #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +27,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldFixShape = true; ma::adapt(in); @@ -39,7 +40,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fusion.cc b/test/fusion.cc index 582188417..f5030511a 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -3,11 +3,11 @@ #include #include #include -#include #include #include #include #include +#include double const a_param = 0.2; double const b_param = 1.0; @@ -139,15 +139,15 @@ struct GroupCode : public Parma_GroupCode apf::Mesh2* mesh; gmi_model* model; const char* meshFile; - void run(int group) + void run(int group, pcu::PCU *PCUObj) { if (group == 0) { - mesh = apf::loadMdsMesh(model, meshFile); + mesh = apf::loadMdsMesh(model, meshFile, PCUObj); mesh->verify(); testIndexing(mesh); fusionAdapt(mesh); } else { - mesh = apf::makeEmptyMdsMesh(model, 2, false); + mesh = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); } } }; @@ -156,17 +156,18 @@ int main( int argc, char* argv[]) { PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); GroupCode code; code.model = makeModel(); code.meshFile = argv[1]; - apf::Unmodulo outMap(code.mesh->getPCU()->Self(), 2); - Parma_SplitPartition(NULL, 2, code); + apf::Unmodulo outMap(pcu_obj.get()->Self(), 2); + Parma_SplitPartition(NULL, 2, code, pcu_obj.get()); apf::remapPartition(code.mesh, outMap); code.mesh->verify(); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fusion2.cc b/test/fusion2.cc index cd06fe2ca..65b4552ea 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -2,17 +2,17 @@ #include #include #include -#include #include #include #include #include +#include -static apf::Mesh2* makeEmptyMesh() +static apf::Mesh2* makeEmptyMesh(pcu::PCU *PCUObj) { /* dont use null model in production */ gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(g, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(g, 2, false, PCUObj); /* both planes create the field at the beginning */ apf::createPackedField(m, "fusion", 6); return m; @@ -90,9 +90,9 @@ static void checkValues(apf::Mesh2* m) struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; - void run(int group) + void run(int group, pcu::PCU *PCUObj) { - mesh = ::makeEmptyMesh(); + mesh = ::makeEmptyMesh(PCUObj); if (group == 0) { addOneTri(mesh); setValues(mesh); @@ -140,17 +140,18 @@ static void globalCode(apf::Mesh2* m) int main( int argc, char* argv[]) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() == 2); + PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 2); gmi_register_null(); GroupCode code; int const groupSize = 1; - apf::Unmodulo outMap(PCU_Comm_Self(), groupSize); - Parma_SplitPartition(NULL, groupSize, code); + apf::Unmodulo outMap(PCUObj.get()->Self(), groupSize); + //Parma_SplitPartition(nullptr, groupSize, code, PCUObj.get()); /* update mesh for leaving groups */ apf::remapPartition(code.mesh, outMap); globalCode(code.mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fusion3.cc b/test/fusion3.cc index a8df775e4..c44ec2a13 100644 --- a/test/fusion3.cc +++ b/test/fusion3.cc @@ -3,13 +3,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include using std::vector; class Expression @@ -283,11 +283,12 @@ int main(int argc, char * argv[]) { PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_model* model = makeModel(); gmi_write_dmg(model, "made.dmg"); - apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1]); + apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1], PCUObj.get()); mesh->verify(); Vortex sfv(mesh, center, modelLen); const ma::Input* in = ma::configure(mesh,&sfv); @@ -296,4 +297,7 @@ int main(int argc, char * argv[]) apf::writeVtkFiles("adapted",mesh); //clean data // to do + apf::destroyMesh(mesh); + } + MPI_Finalize(); } diff --git a/test/gap.cc b/test/gap.cc index 2b1e46af5..11fe1c72a 100644 --- a/test/gap.cc +++ b/test/gap.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #endif #include #include +#include namespace { apf::MeshTag* setWeights(apf::Mesh* m) { @@ -30,10 +30,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 5); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -47,7 +48,7 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setWeights(m); int verbose = 2; // set to 1 to silence the 'endStep' stats @@ -68,6 +69,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/generate.cc b/test/generate.cc index 523edf3b5..f72ce3c43 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include #include //cout #include //option parser @@ -73,9 +73,9 @@ void messageHandler(int type, const char* msg) } } -pParMesh generate(pGModel mdl, std::string meshCaseName) { +pParMesh generate(pGModel mdl, std::string meshCaseName, pcu::PCU *PCUObj) { pAManager attmngr = SModel_attManager(mdl); - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) fprintf(stdout, "Loading mesh case %s...\n", meshCaseName.c_str()); pACase mcaseFile = AMAN_findCase(attmngr, meshCaseName.c_str()); PCU_ALWAYS_ASSERT(mcaseFile); @@ -101,16 +101,16 @@ pParMesh generate(pGModel mdl, std::string meshCaseName) { if( !disable_surface ) { const double stime = MPI_Wtime(); - if(0==PCU_Comm_Self()) { + if(0==PCUObj->Self()) { printf("Meshing surface..."); fflush(stdout); } pSurfaceMesher surfaceMesher = SurfaceMesher_new(mcase, pmesh); SurfaceMesher_execute(surfaceMesher, NULL); SurfaceMesher_delete(surfaceMesher); - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) printf(" %f seconds\n", MPI_Wtime()-stime); if( ! surfaceMeshFile.empty() ) { - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) printf(" writing surface mesh %s\n", surfaceMeshFile.c_str()); PM_write(pmesh, surfaceMeshFile.c_str(), NULL); } @@ -118,20 +118,20 @@ pParMesh generate(pGModel mdl, std::string meshCaseName) { if( !disable_volume ) { const double vtime = MPI_Wtime(); - if(0==PCU_Comm_Self()) { + if(0==PCUObj->Self()) { printf("Meshing volume..."); fflush(stdout); } pVolumeMesher volumeMesher = VolumeMesher_new(mcase, pmesh); VolumeMesher_execute(volumeMesher, NULL); VolumeMesher_delete(volumeMesher); - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) printf(" %f seconds\n", MPI_Wtime()-vtime); } return pmesh; } -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { opterr = 0; static struct option long_opts[] = { @@ -164,7 +164,7 @@ void getConfig(int argc, char** argv) { if (c == -1) break; //end of options switch (c) { case 0: // enable-log|disable-volume|disable-surf - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf ("read arg %d\n", c); break; case 'n': @@ -174,11 +174,11 @@ void getConfig(int argc, char** argv) { surfaceMeshFile = std::string(optarg); break; case '?': - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf ("warning: skipping unrecognized option\n"); break; default: - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -186,7 +186,7 @@ void getConfig(int argc, char** argv) { if(argc-optind != 2) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -195,7 +195,7 @@ void getConfig(int argc, char** argv) { outMeshFile = caseName = std::string(argv[i++]); outMeshFile.append("/"); - if (!PCU_Comm_Self()) { + if (!PCUObj->Self()) { std::cout << "enable_log " << should_log << " disable_surface " << disable_surface << " disable_volume " << disable_volume << @@ -249,7 +249,7 @@ bool hasExtension(std::string s, std::string ext) { } #endif -pNativeModel loadNativeModel() { +pNativeModel loadNativeModel(pcu::PCU *PCUObj) { enum { TEXT_FORMAT = 0 }; pNativeModel nm = 0; if ( nativeModelFile.empty() ) { @@ -257,18 +257,18 @@ pNativeModel loadNativeModel() { #ifdef SIM_ACIS } else if (hasExtension(nativeModelFile, "sat")) { nm = AcisNM_createFromFile(nativeModelFile.c_str(), TEXT_FORMAT); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("loaded acis native model\n"); #endif #ifdef SIM_PARASOLID } else if (hasExtension(nativeModelFile, "xmt_txt") || hasExtension(nativeModelFile, "x_t") ) { nm = ParasolidNM_createFromFile(nativeModelFile.c_str(), TEXT_FORMAT); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("loaded parasolid native model\n"); #endif } else { - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("native model file has bad extension"); exit(EXIT_FAILURE); } @@ -316,27 +316,28 @@ void simStop() { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); pcu::Protect(); - getConfig(argc,argv); + getConfig(argc,argv,PCUObj.get()); if (should_attach_order && should_fix_pyramids) { - if (!PCU_Comm_Self()) + if (!PCUObj.get()->Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = 0; } simStart(); - pNativeModel nm = loadNativeModel(); + pNativeModel nm = loadNativeModel(PCUObj.get()); pGModel simModel = GM_load(modelFile.c_str(), nm, NULL); const double t0 = MPI_Wtime(); - pParMesh sim_mesh = generate(simModel, caseName); + pParMesh sim_mesh = generate(simModel, caseName, PCUObj.get()); const double t1 = MPI_Wtime(); - if(!PCU_Comm_Self()) + if(!PCUObj.get()->Self()) printf("Mesh generated in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, PCUObj.get()); if (should_attach_order) attachOrder(simApfMesh); gmi_register_sim(); @@ -353,6 +354,6 @@ int main(int argc, char** argv) simStop(); Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/poisson.cc b/test/poisson.cc index 281d6f194..874ad03ad 100644 --- a/test/poisson.cc +++ b/test/poisson.cc @@ -58,7 +58,7 @@ class Poisson { } void run(pcu::PCU *PCUObj) { - setup_grid(pcu::PCU *PCUObj); + setup_grid(PCUObj); setup_lin_alg(); fill_volumetric(); fill_boundary(); diff --git a/test/repartition.cc b/test/repartition.cc index c8174cbad..98caca57e 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -98,6 +98,7 @@ void balance(apf::Mesh2* m) int main(int argc, char** argv) { MPI_Init(&argc,&argv); + { auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); @@ -114,7 +115,7 @@ int main(int argc, char** argv) Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); - + } MPI_Finalize(); } diff --git a/test/serialize.cc b/test/serialize.cc index 77f25b2e9..3a40c2663 100644 --- a/test/serialize.cc +++ b/test/serialize.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -13,12 +12,13 @@ #endif #include #include +#include struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; const char* meshFile; - void run(int) + void run(int, pcu::PCU*) { mesh->writeNative(meshFile); } @@ -27,10 +27,11 @@ struct GroupCode : public Parma_GroupCode int main( int argc, char* argv[]) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -46,7 +47,7 @@ int main( int argc, char* argv[]) gmi_register_null(); crv::getBezier(2);//hack to make sure curved meshes can be serialized! GroupCode code; - code.mesh = apf::loadMdsMesh(argv[1], argv[2]); + code.mesh = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); code.meshFile = argv[3]; apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); Parma_ShrinkPartition(code.mesh, atoi(argv[4]), code); @@ -58,7 +59,7 @@ int main( int argc, char* argv[]) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/spr_test.cc b/test/spr_test.cc index 6d16041c6..6be086356 100644 --- a/test/spr_test.cc +++ b/test/spr_test.cc @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { @@ -19,10 +19,11 @@ int main(int argc, char** argv) const char* outFile = argv[3]; const int order = atoi(argv[4]); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile); + apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile, PCUObj.get()); if (mesh->findTag("coordinates_edg")) mesh->changeShape(apf::getSerendipity(), false); apf::Field* f = @@ -30,13 +31,13 @@ int main(int argc, char** argv) apf::Field* eps = spr::getGradIPField(f, "eps", order); apf::destroyField(f); double adaptRatio = 0.1; - apf::Field* sizef = spr::getSPRSizeField(eps,adaptRatio); + apf::Field* sizef = spr::getSPRSizeField(eps,adaptRatio,PCUObj.get()); apf::destroyField(eps); writeVtkFiles(outFile,mesh); apf::destroyField(sizef); mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } From 0db8e21a60740381ae5ca0faf0ef1e0650f2e353 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 17 Apr 2024 10:26:10 -0400 Subject: [PATCH 102/141] loadMdsPart take optional PCU --- mds/apfMDS.cc | 8 ++++++-- mds/apfMDS.h | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 8fd0b7327..da696372b 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -1407,10 +1407,14 @@ void hackMdsAdjacency(Mesh2* in, MeshEntity* up, int i, MeshEntity* down) mds_hack_adjacent(&m->mesh->mds, fromEnt(up), i, fromEnt(down)); } -Mesh2* loadMdsPart(gmi_model* model, const char* meshfile) +Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { MeshMDS* m = new MeshMDS(); - m->init(apf::getLagrange(1)); + if(PCUObj != nullptr){ + m->init(apf::getLagrange(1), PCUObj); + } else { + m->init(apf::getLagrange(1)); + } m->mesh = mds_read_smb2(m->getPCU()->GetCHandle(), model, meshfile, 1, m); m->isMatched = false; m->ownsModel = true; diff --git a/mds/apfMDS.h b/mds/apfMDS.h index 2d14e9917..4a63dd328 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -218,9 +218,9 @@ Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCM int gmshMajorVersion(const char* filename); -Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename); +Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj = nullptr); -Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename); +Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename, pcu::PCU *PCUObj = nullptr); Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename); @@ -244,7 +244,7 @@ void disownMdsModel(Mesh2* in); void setMdsMatching(Mesh2* in, bool has); -Mesh2* loadMdsPart(gmi_model* model, const char* meshfile); +Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj = nullptr); void writeMdsPart(Mesh2* m, const char* meshfile); } From 6f4c6df185fb769b8536e453eb02b8bdab98ebff Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 17 Apr 2024 10:28:11 -0400 Subject: [PATCH 103/141] loadMdsFromGmsh to take PCU --- mds/mdsGmsh.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc index 4d0ce7875..b6d3f8524 100644 --- a/mds/mdsGmsh.cc +++ b/mds/mdsGmsh.cc @@ -459,17 +459,17 @@ void gmshFindDmg(const char* fnameDmg, const char* filename) } -Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename) +Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false); + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); readGmsh(m, filename); return m; } -Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename) +Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename, pcu::PCU *PCUObj) { gmshFindDmg(fnameDmg, filename); // new function that scans $Entities and writes a dmg - Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false); + Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); readGmsh(m, filename); return m; } From 9900b46f20a920163f35b7bc6ff181e02a54c277 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 17 Apr 2024 10:30:04 -0400 Subject: [PATCH 104/141] fixed runInGroups() to work with new PCU --- parma/group/parma_group.cc | 64 +++++++++++++++++++++++++++++--------- parma/parma.h | 8 +++-- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index 0c0e9487b..3cbbdfe68 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -1,4 +1,5 @@ #include +#include #include using apf::Remap; @@ -27,26 +28,41 @@ typedef Parma_GroupCode GroupCode; static void runInGroups( apf::Mesh2* m, + pcu::PCU *PCUObj, Remap& inMap, Remap& groupMap, Remap& outMap, - GroupCode& code) + GroupCode& code + ) { - int self = m->getPCU()->Self(); + int self; + pcu::PCU* expandedPCU; + if(m != nullptr){ + self = m->getPCU()->Self(); + expandedPCU = m->getPCU(); + } else { + self = PCUObj->Self(); + expandedPCU = PCUObj; + } + int groupRank = inMap(self); int group = groupMap(self); - auto* expandedPCU = m->getPCU(); + MPI_Comm groupComm; MPI_Comm_split(expandedPCU->GetMPIComm(), group, groupRank, &groupComm); auto groupedPCU = std::unique_ptr(new pcu::PCU(groupComm)); - m->switchPCU(groupedPCU.get()); - if (m) + if (m){ + m->switchPCU(groupedPCU.get()); apf::remapPartition(m, inMap); + } + code.PCUObj = std::move(groupedPCU); code.run(group); - m->switchPCU(expandedPCU); + MPI_Comm_free(&groupComm); - if (m) + if (m){ + m->switchPCU(expandedPCU); apf::remapPartition(m, outMap); + } } struct RetreatCode : public GroupCode @@ -80,28 +96,48 @@ static void retreatToGroup( Remap& retreatMap, Remap& groupMap, Remap& outMap, - GroupCode& code) + GroupCode& code, + pcu::PCU *PCUObj) { retreat(m, retreatMap); RetreatCode retreatCode(code, m, factor); - runInGroups(m, inMap, groupMap, outMap, retreatCode); + runInGroups(m, PCUObj, inMap, groupMap, outMap, retreatCode); m->migrate(retreatCode.plan); } -void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun) +void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj) { + if(m == nullptr){ + PCU_ALWAYS_ASSERT(PCUObj != nullptr); + } + if(m != nullptr && PCUObj != nullptr){ + PCU_ALWAYS_ASSERT(m->getPCU() == PCUObj); + } apf::Divide inMap(factor); apf::Modulo groupMap(factor); apf::Round retreatMap(factor); apf::Multiply outMap(factor); - retreatToGroup(m, factor, inMap, retreatMap, groupMap, outMap, toRun); + retreatToGroup(m, factor, inMap, retreatMap, groupMap, outMap, toRun, PCUObj); } -void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun) +void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj) { + if(m == nullptr){ + PCU_ALWAYS_ASSERT(PCUObj != nullptr); + } + if(m != nullptr && PCUObj != nullptr){ + PCU_ALWAYS_ASSERT(m->getPCU() == PCUObj); + } apf::Modulo inMap(factor); apf::Divide groupMap(factor); - apf::Unmodulo outMap(m->getPCU()->Self(), factor); - runInGroups(m, inMap, groupMap, outMap, toRun); + if(m != nullptr){ + apf::Unmodulo outMap(m->getPCU()->Self(), factor); + runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); + } else { + apf::Unmodulo outMap(PCUObj->Self(), factor); + runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); + } + + //runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); } diff --git a/parma/parma.h b/parma/parma.h index 3d65c9779..b9fe005e2 100644 --- a/parma/parma.h +++ b/parma/parma.h @@ -13,6 +13,7 @@ #include "apf.h" #include "apfPartition.h" +#include /** * @brief get entity imbalance @@ -288,6 +289,7 @@ apf::MeshTag* Parma_WeighByMemory(apf::Mesh* m); */ struct Parma_GroupCode { + std::unique_ptr PCUObj; /** * @brief Called withing sub-groups. * @details Within a group, all PCU functions behave @@ -299,7 +301,7 @@ struct Parma_GroupCode * the group. * @param group the group id number, starting from zero */ - virtual void run(int group, pcu::PCU *PCUObj) = 0; + virtual void run(int group) = 0; }; /** @@ -314,7 +316,7 @@ struct Parma_GroupCode * parts are combined into one and then that one is split back * out into (factor) contiguous part ids again. */ -void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); +void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj = nullptr); /** * @brief Split the processes into groups of (factor). @@ -327,7 +329,7 @@ void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); * apf::remapPartition is used to maintain the mesh structure * during these transitions. */ -void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj); +void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj = nullptr); /** * @brief Compute maximal independent set numbering From b0a6ff227aefc208c4d9c6aed980c8f7e620cae1 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 17 Apr 2024 10:31:10 -0400 Subject: [PATCH 105/141] reverted the run function in GroupCode structs to not take a PCU --- phasta/condense.cc | 2 +- phasta/phAdapt.cc | 4 ++-- phasta/phCook.cc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/phasta/condense.cc b/phasta/condense.cc index f7d9e2fc4..21e5e95d5 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -21,7 +21,7 @@ namespace { struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; ph::Input ctrl; - void run(int, pcu::PCU*) { + void run(int) { apf::reorderMdsMesh(mesh,NULL); chef::preprocess(mesh,ctrl); } diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index 29e99256c..a40ada8c4 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -55,7 +55,7 @@ struct AdaptCallback : public Parma_GroupCode : mesh(m), field(szfld), in(NULL) { } AdaptCallback(apf::Mesh2* m, apf::Field* szfld, ph::Input* inp) : mesh(m), field(szfld), in(inp) { } - void run(int, pcu::PCU*) { + void run(int) { ma::Input* ma_in = ma::makeAdvanced(ma::configure(mesh, field)); if( in ) { //chef defaults @@ -125,7 +125,7 @@ void adaptShrunken(apf::Mesh2* m, double minPartDensity, if (!m->getPCU()->Self()) lion_eprint(1,"adaptShrunken limit set to %f factor computed as %d\n", minPartDensity, factor); if (factor == 1) { - callback.run(0, m->getPCU()); + callback.run(0); } else { warnAboutShrinking(factor, m->getPCU()); Parma_ShrinkPartition(m, factor, callback); diff --git a/phasta/phCook.cc b/phasta/phCook.cc index f4db9f7be..7dfbfdb8b 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -248,7 +248,7 @@ namespace { ph::Input* input; ph::BCs* boundary; apf::Mesh2* mesh; - void run(int, pcu::PCU*) { + void run(int) { ph::Output groupOut; //streaming not supported from group code! groupOut.openfile_write = chef::openfile_write; From 52c9ab8fb49bb9ebb67b28ba8f7e6dd2d77dbb21 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 17 Apr 2024 10:32:06 -0400 Subject: [PATCH 106/141] test files fixed to use PCUObj; all pass --- test/H1Shapes.cc | 15 ++++----- test/L2Shapes.cc | 15 ++++----- test/capVol.cc | 13 ++++---- test/collapse.cc | 3 +- test/fusion.cc | 9 +++--- test/fusion2.cc | 9 +++--- test/ghost.cc | 15 ++++----- test/ghostEdge.cc | 9 +++--- test/ghostMPAS.cc | 9 +++--- test/gmsh.cc | 15 ++++----- test/graphdist.cc | 11 ++++--- test/hierarchic.cc | 11 ++++--- test/highOrderSizeFields.cc | 21 +++++++------ test/highOrderSolutionTransfer.cc | 19 ++++++----- test/icesheet.cc | 9 +++--- test/inClosureOf_test.cc | 7 +++-- test/intrude.cc | 11 ++++--- test/loadPart.cc | 13 ++++---- test/ma_insphere.cc | 9 +++--- test/ma_test.cc | 9 +++--- test/ma_test_analytic_model.cc | 14 ++++----- test/makeAllCavities.cc | 23 +++++++------- test/matchedNodeElmReader.cc | 46 ++++++++++++++------------- test/measureAnisoStats.cc | 20 ++++++------ test/measureIsoStats.cc | 21 +++++++------ test/mixedNumbering.cc | 1 - test/mkmodel.cc | 11 ++++--- test/mktopomodel.cc | 11 ++++--- test/modelInfo.cc | 11 ++++--- test/moving.cc | 9 +++--- test/nedelecShapes.cc | 11 ++++--- test/nektar_align.cc | 9 +++--- test/neper.cc | 11 ++++--- test/newdim.cc | 9 +++--- test/osh2smb.cc | 12 +++---- test/poisson.cc | 2 +- test/print_pumipic_partition.cc | 15 ++++----- test/pumi.cc | 47 ++++++++++++++-------------- test/pumiLoadMesh.cc | 10 +++--- test/quality.cc | 25 ++++++++------- test/refine2x.cc | 11 ++++--- test/render.cc | 11 ++++--- test/renderClass.cc | 11 ++++--- test/render_ascii.cc | 11 ++++--- test/reorder.cc | 11 ++++--- test/reposition.cc | 11 ++++--- test/residualErrorEstimation_test.cc | 9 +++--- test/rm_extrusion.cc | 29 ++++++++--------- test/runSimxAnisoAdapt.cc | 24 +++++++------- test/scale.cc | 15 ++++----- test/serialize.cc | 2 +- test/shapefun.cc | 23 +++++++------- test/shapefun2.cc | 15 ++++----- test/simZBalance.cc | 42 +++++++++++-------------- test/sim_part.cc | 11 ++++--- test/smb2osh.cc | 12 +++---- test/snap.cc | 9 +++--- test/swapDoubles.cc | 9 ++++-- test/test_integrator.cc | 9 +++--- test/test_matrix_grad.cc | 9 +++--- test/test_scaling.cc | 9 +++--- test/test_verify.cc | 9 +++--- test/tetrahedronize.cc | 9 +++--- test/torus_ma_test.cc | 9 +++--- test/ugridptnstats.cc | 7 +++-- test/uniform.cc | 15 ++++----- test/verify_2nd_order_shapes.cc | 11 ++++--- 67 files changed, 482 insertions(+), 421 deletions(-) diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc index f65899074..e45ba32e3 100644 --- a/test/H1Shapes.cc +++ b/test/H1Shapes.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -14,6 +13,7 @@ #include #include #include +#include // User defined vector functions E(x,y,z) of order up to 6 @@ -29,12 +29,13 @@ void testH1( int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -44,12 +45,12 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],PCUObj.get()); m->verify(); // test fields interpolating a user-defined vector field for (int i = 1; i <= 6; i++) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) lion_oprint(1, "----TESTING VECTOR FIELD OF ORDER %d----\n", i); testH1( m, /* mesh */ @@ -61,7 +62,7 @@ int main(int argc, char** argv) // test fields interpolating a user-defined matrix field for (int i = 1; i <= 6; i++) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) lion_oprint(1, "----TESTING MATRIX FIELD OF ORDER %d----\n", i); testH1( m, /* mesh */ @@ -72,7 +73,7 @@ int main(int argc, char** argv) } apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc index d8af11975..cbe2a4a12 100644 --- a/test/L2Shapes.cc +++ b/test/L2Shapes.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -14,6 +13,7 @@ #include #include #include +#include using namespace std; @@ -32,12 +32,13 @@ void testL2writeNative( int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -47,7 +48,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],PCUObj.get()); m->verify(); for (int i = 0; i <= 6; i++) { @@ -61,7 +62,7 @@ int main(int argc, char** argv) testL2writeNative(m, 3, 3); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -204,10 +205,10 @@ void testL2writeNative( // 2- read the mesh back in and make sure fields are on the mesh // 3- clean up the newly loaded mesh m->writeNative("L2Shape_test_mesh.smb"); - apf::Mesh2* m2 = apf::loadMdsMesh(".null", "./L2Shape_test_mesh.smb"); + apf::Mesh2* m2 = apf::loadMdsMesh(".null", "./L2Shape_test_mesh.smb", m->getPCU()); int fCount = 0; for (int i = 0; i < m2->countFields(); i++) { - if(0==m->getPCU()->Self()) + if(0==m2->getPCU()->Self()) lion_oprint(1, "field %d's name and shape are %s and %s\n", i, m2->getField(i)->getName(), m2->getField(i)->getShape()->getName()); fCount++; diff --git a/test/capVol.cc b/test/capVol.cc index 87dedd34b..9f9c0b8c5 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -1,11 +1,11 @@ #include #include -#include // Output #include // Parallelism +#include #include // Mesh interfaces @@ -30,6 +30,7 @@ namespace { void myExit(int exit_code = EXIT_SUCCESS) { gmi_cap_stop(); + PCU_Comm_Free(); MPI_Finalize(); exit(exit_code); } @@ -76,8 +77,7 @@ void printUsage(char *argv0) { int main(int argc, char** argv) { // Initialize parallelism. MPI_Init(&argc, &argv); - { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + PCU_Comm_Init(); // Initialize logging. lion_set_stdout(stdout); @@ -85,7 +85,7 @@ int main(int argc, char** argv) { // Check arguments or print usage. if (argc < 3) { - if (pcu_obj.get()->Self() == 0) { + if (PCU_Comm_Self() == 0) { printUsage(argv[0]); } myExit(EXIT_FAILURE); @@ -271,7 +271,6 @@ int main(int argc, char** argv) { // Exit calls. gmi_cap_stop(); - } + PCU_Comm_Free(); MPI_Finalize(); -} - +} \ No newline at end of file diff --git a/test/collapse.cc b/test/collapse.cc index 1eb05333a..f989c58d3 100644 --- a/test/collapse.cc +++ b/test/collapse.cc @@ -2,7 +2,6 @@ #include #include #include -//#include #include #ifdef HAVE_SIMMETRIX #include @@ -22,7 +21,7 @@ namespace { struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; - void run(int, pcu::PCU*) { + void run(int) { mesh->writeNative(outFile); } }; diff --git a/test/fusion.cc b/test/fusion.cc index f5030511a..7c341fea3 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -139,15 +139,15 @@ struct GroupCode : public Parma_GroupCode apf::Mesh2* mesh; gmi_model* model; const char* meshFile; - void run(int group, pcu::PCU *PCUObj) + void run(int group) { if (group == 0) { - mesh = apf::loadMdsMesh(model, meshFile, PCUObj); + mesh = apf::loadMdsMesh(model, meshFile, PCUObj.get()); mesh->verify(); testIndexing(mesh); fusionAdapt(mesh); } else { - mesh = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); + mesh = apf::makeEmptyMdsMesh(model, 2, false, PCUObj.get()); } } }; @@ -163,7 +163,8 @@ int main( int argc, char* argv[]) code.model = makeModel(); code.meshFile = argv[1]; apf::Unmodulo outMap(pcu_obj.get()->Self(), 2); - Parma_SplitPartition(NULL, 2, code, pcu_obj.get()); + Parma_SplitPartition(nullptr, 2, code, pcu_obj.get()); + code.mesh->switchPCU(pcu_obj.get()); apf::remapPartition(code.mesh, outMap); code.mesh->verify(); code.mesh->destroyNative(); diff --git a/test/fusion2.cc b/test/fusion2.cc index 65b4552ea..511866a34 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -90,9 +90,9 @@ static void checkValues(apf::Mesh2* m) struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; - void run(int group, pcu::PCU *PCUObj) + void run(int group) { - mesh = ::makeEmptyMesh(PCUObj); + //mesh = ::makeEmptyMesh(PCUObj); if (group == 0) { addOneTri(mesh); setValues(mesh); @@ -146,11 +146,12 @@ int main( int argc, char* argv[]) PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 2); gmi_register_null(); GroupCode code; + code.mesh = makeEmptyMesh(PCUObj.get()); int const groupSize = 1; apf::Unmodulo outMap(PCUObj.get()->Self(), groupSize); - //Parma_SplitPartition(nullptr, groupSize, code, PCUObj.get()); + Parma_SplitPartition(code.mesh, groupSize, code, PCUObj.get()); /* update mesh for leaving groups */ - apf::remapPartition(code.mesh, outMap); + //apf::remapPartition(code.mesh, outMap); globalCode(code.mesh); } MPI_Finalize(); diff --git a/test/ghost.cc b/test/ghost.cc index d225f2221..6b1b59b5b 100644 --- a/test/ghost.cc +++ b/test/ghost.cc @@ -2,11 +2,11 @@ #include #include #include -#include #include #include #include #include +#include namespace { const char* modelFile = 0; @@ -18,10 +18,10 @@ namespace { apf::destroyMesh(m); } - void getConfig(int argc, char** argv) + void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -78,16 +78,17 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); apf::MeshTag* weights = applyFun3dWeight(m); runParma(m,weights); m->destroyTag(weights); Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ghostEdge.cc b/test/ghostEdge.cc index fbacc304a..c91ef9929 100644 --- a/test/ghostEdge.cc +++ b/test/ghostEdge.cc @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include namespace { const char* modelFile = 0; @@ -51,14 +51,15 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); runParma(m); m->writeNative(argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ghostMPAS.cc b/test/ghostMPAS.cc index 8e1ea1715..e908b4528 100644 --- a/test/ghostMPAS.cc +++ b/test/ghostMPAS.cc @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include namespace { const char* modelFile = 0; @@ -53,14 +53,15 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); runParma(m); m->writeNative(argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/gmsh.cc b/test/gmsh.cc index b2f3911ec..999f6215e 100644 --- a/test/gmsh.cc +++ b/test/gmsh.cc @@ -3,18 +3,19 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n" "The input .msh and output .smb file names are required. \n" "If 'none' is specified as the input model file name then \n" @@ -39,22 +40,22 @@ int main(int argc, char** argv) apf::Mesh2* m = NULL; if (gmshVersion == 2) { if (model.compare("none") == 0) { - m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str()); + m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str(), PCUObj.get()); apf::deriveMdsModel(m); gmi_write_dmg(m->getModel(),outModel.c_str()); } else { - m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str()); + m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str(), PCUObj.get()); } } else if (gmshVersion == 4) { if (model.compare("none") == 0) { - m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str()); + m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str(), PCUObj.get()); } } m->verify(); m->writeNative(outMesh.c_str()); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/graphdist.cc b/test/graphdist.cc index 98fd857a9..7834a93c7 100644 --- a/test/graphdist.cc +++ b/test/graphdist.cc @@ -2,25 +2,26 @@ #include #include #include -#include #include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); dsp::Boundary moving; moving.insert(m->findModelEntity(2, 57)); moving.insert(m->findModelEntity(2, 62)); @@ -31,6 +32,6 @@ int main(int argc, char** argv) apf::writeVtkFiles(argv[3], m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/hierarchic.cc b/test/hierarchic.cc index 50e5bf24d..2dbec0134 100644 --- a/test/hierarchic.cc +++ b/test/hierarchic.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -9,6 +8,7 @@ #include #include #include +#include namespace { @@ -194,17 +194,18 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(! PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(! pcu_obj.get()->Self()); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); apf::reorderMdsMesh(m); m->verify(); int p_order = atoi(argv[3]); test(m, p_order); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc index 7cbbe34fc..7e4806299 100644 --- a/test/highOrderSizeFields.cc +++ b/test/highOrderSizeFields.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -20,6 +19,7 @@ #include #include #include +#include void computeSizesFrames( @@ -31,12 +31,14 @@ void computeSizesFrames( void testAdapt( const char* model, const char* mesh, - int order); + int order, + pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX @@ -49,19 +51,19 @@ int main(int argc, char** argv) gmi_register_mesh(); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr <<"usage: " << argv[0] << " \n"; return EXIT_FAILURE; } for (int order = 3; order < 5; order++) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) lion_oprint(1, "Testing aniso adapt w/ sizefield order %d\n", order); - testAdapt(argv[1], argv[2], order); + testAdapt(argv[1], argv[2], order, PCUObj.get()); } - PCU_Comm_Free(); + } #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); @@ -113,10 +115,11 @@ void computeSizesFrames( void testAdapt( const char* model, const char* mesh, - int order) + int order, + pcu::PCU *PCUObj) { gmi_model* g = gmi_load(model); - apf::Mesh2* m = apf::loadMdsMesh(g,mesh); + apf::Mesh2* m = apf::loadMdsMesh(g,mesh,PCUObj); m->verify(); const ma::Input* in = ma::configureUniformRefine(m, 1, 0); diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index bba9616cd..f5ad96c41 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -17,6 +16,7 @@ #include #include #include +#include void E_exact(const apf::Vector3& x, apf::Vector3& value, int p); @@ -33,6 +33,7 @@ double testH1Field( void testCurveAdapt( const char* modelFile, const char* meshFile, + pcu::PCU *PCUObj, const int mesh_order, const int exact_order, const int field_order); @@ -42,7 +43,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -62,28 +64,28 @@ int main(int argc, char** argv) */ // linear adapt - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, PCUObj.get(), 1 /*mesh_order*/, 2 /*exact_order*/, 2 /*field_order*/); // quadratic adapts - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, PCUObj.get(), 2 /*mesh_order*/, 2 /*exact_order*/, 4 /*field_order*/); - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, PCUObj.get(), 2 /*mesh_order*/, 3 /*exact_order*/, 6 /*field_order*/); // cubic adapt - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, PCUObj.get() 3 /*mesh_order*/, 2 /*exact_order*/, 6 /*field_order*/); - PCU_Comm_Free(); + } #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); @@ -193,12 +195,13 @@ double testH1Field( void testCurveAdapt( const char* modelFile, const char* meshFile, + pcu::PCU *PCUObj, const int mesh_order, const int exact_order, const int field_order) { - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); m->verify(); if (mesh_order > 1) { diff --git a/test/icesheet.cc b/test/icesheet.cc index ac2decfb7..fe627f31a 100644 --- a/test/icesheet.cc +++ b/test/icesheet.cc @@ -4,11 +4,11 @@ #include #include #include -#include #include #include #include #include +#include /* tags on vertices */ #define INTERIORTAG 0 @@ -483,7 +483,8 @@ int main(int argc, char** argv) } MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -494,7 +495,7 @@ int main(int argc, char** argv) readMesh(argv[1],m); const int dim = 3; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false, PCUObj.get()); apf::GlobalToVert outMap; apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap); delete [] m.elements; @@ -524,6 +525,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/inClosureOf_test.cc b/test/inClosureOf_test.cc index 7b673dee3..a7b60aff7 100644 --- a/test/inClosureOf_test.cc +++ b/test/inClosureOf_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #endif #include #include +#include int main(int argc, char** argv) { @@ -20,7 +20,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -87,7 +88,7 @@ int main(int argc, char** argv) gmi_destroy(model); // deleting the model - PCU_Comm_Free(); + } #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); diff --git a/test/intrude.cc b/test/intrude.cc index 3e5fb6cf2..e79a302b8 100644 --- a/test/intrude.cc +++ b/test/intrude.cc @@ -2,25 +2,26 @@ #include #include #include -#include #include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); ma::ModelExtrusions extrusions; extrusions.push_back(ma::ModelExtrusion( m->findModelEntity(1, 2), @@ -41,6 +42,6 @@ int main(int argc, char** argv) m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/loadPart.cc b/test/loadPart.cc index 902ed0dbf..fb6395316 100644 --- a/test/loadPart.cc +++ b/test/loadPart.cc @@ -3,19 +3,20 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() == 1); + PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Load a single part from a partitioned mesh and " "write it as a serial part.\n" "Usage: %s \n", argv[0]); @@ -25,13 +26,13 @@ int main(int argc, char** argv) gmi_register_null(); gmi_register_mesh(); gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsPart(g, argv[1]); + apf::Mesh2* m = apf::loadMdsPart(g, argv[1], PCUObj.get()); apf::deriveMdsModel(m); gmi_write_dmg(g, argv[3]); m->writeNative(argv[2]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ma_insphere.cc b/test/ma_insphere.cc index 1fd0dc739..6319e94af 100644 --- a/test/ma_insphere.cc +++ b/test/ma_insphere.cc @@ -1,10 +1,10 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { @@ -30,7 +30,8 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(apf::getDeterminant(matrix) == -1485); // Test insphere (create a mesh with one tet) - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); apf::Vector3 a(0, 0, 0); apf::Vector3 b(-6, 0, 0); @@ -39,7 +40,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, 3, true); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, 3, true, PCUObj.get()); apf::ModelEntity* m = mesh->findModelEntity(0, 0); apf::MeshEntity* v[4]; for (int i=0; i<4; i++) { @@ -57,6 +58,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ma_test.cc b/test/ma_test.cc index 8d4558f24..4d60d00be 100644 --- a/test/ma_test.cc +++ b/test/ma_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -12,6 +11,7 @@ #include #endif #include +#include class Linear : public ma::IsotropicFunction { @@ -45,7 +45,8 @@ int main(int argc, char** argv) const char* layerTagString = (argc==4) ? argv[3] : ""; const double adaptRefineFactor = (argc==5) ? atoi(argv[4]) : 3; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -55,7 +56,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); Linear sf(m,adaptRefineFactor); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); @@ -80,7 +81,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ma_test_analytic_model.cc b/test/ma_test_analytic_model.cc index 6d5d45635..b12d01317 100644 --- a/test/ma_test_analytic_model.cc +++ b/test/ma_test_analytic_model.cc @@ -9,11 +9,11 @@ #include #include #include -#include #include #include #include +#include const double pi = 3.14159265359; @@ -65,10 +65,10 @@ gmi_model* makeSphere() return model; } -apf::Mesh2* createSphereMesh() +apf::Mesh2* createSphereMesh(pcu::PCU *PCUObj) { gmi_model* model = makeSphere(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* allVs[5]; apf::Vector3 p0( cos(0.), sin(0.), 0.); apf::Vector3 p1( cos(2.*pi/3.), sin(2.*pi/3.), 0.); @@ -171,10 +171,11 @@ apf::Mesh2* createSphereMesh() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - apf::Mesh2* m = createSphereMesh(); + apf::Mesh2* m = createSphereMesh(PCUObj.get()); m->verify(); apf::writeVtkFiles("initial_mesh_on_analytic_model", m); @@ -186,7 +187,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc index 3e11d44fb..6328a7ddc 100644 --- a/test/makeAllCavities.cc +++ b/test/makeAllCavities.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -13,6 +12,7 @@ #include #include #include +#include #ifdef HAVE_SIMMETRIX #include #include @@ -63,8 +63,9 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - if (PCU_Comm_Peers() > 1) { + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + if (PCUObj.get()->Peers() > 1) { printf("%s should only be used for serial (single part) meshes!\n", argv[0]); printf("use the serialize utility to get a serial mesh, and retry!\n"); MPI_Finalize(); @@ -111,7 +112,7 @@ int main(int argc, char** argv) // load the mesh and check if the tag exists on the mesh - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); if (mode.compare(std::string("aa")) == 0) tag = tagMesh(m, -1, 1); @@ -253,7 +254,7 @@ int main(int argc, char** argv) MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -403,8 +404,8 @@ static void makeEntMeshes( } if (m->getType(e) == apf::Mesh::EDGE) { - entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); - entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, m->getPCU()); + entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, m->getPCU()); apf::MeshEntity* vs[2]; m->getDownward(e, 0, vs); apf::Vector3 p[2]; @@ -443,8 +444,8 @@ static void makeEntMeshes( } if (m->getType(e) == apf::Mesh::TRIANGLE) { - entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); - entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); + entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, m->getPCU()); + entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, m->getPCU()); apf::MeshEntity* downverts[3]; apf::MeshEntity* downedges[3]; m->getDownward(e, 0, downverts); @@ -657,8 +658,8 @@ static void makeCavityMeshes( // we do this in a bottom up fashion, ie vets first then edges, faces and tets /* cavityMeshLinear = apf::makeEmptyMdsMesh(m->getModel(), dim, false); */ /* cavityMeshCurved = apf::makeEmptyMdsMesh(m->getModel(), dim, false); */ - cavityMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false); - cavityMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false); + cavityMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false, m->getPCU()); + cavityMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false, m->getPCU()); // verts for (int i = 0; i < (int) icavity0.size(); i++) { diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index d0ae8b1d1..ebab72c5a 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include /* from https://github.com/SCOREC/core/issues/205 0=fully interior of the volume @@ -158,7 +158,7 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC mesh->end(it); } -void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxClass) { +void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) { (void)model; (void)mesh; (void)vtxClass; @@ -426,9 +426,9 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) { void getLocalRange(apf::Gid total, int& local, - apf::Gid& first, apf::Gid& last) { - const int self = PCU_Comm_Self(); - const int peers = PCU_Comm_Peers(); + apf::Gid& first, apf::Gid& last, pcu::PCU *PCUObj) { + const int self = PCUObj->Self(); + const int peers = PCUObj->Peers(); local = total/peers; if( self == peers-1 ) { //last rank apf::Gid lp=local*peers; @@ -438,7 +438,7 @@ void getLocalRange(apf::Gid total, int& local, local += lpd; } } - first = PCU_Exscan_Long(local); + first = PCUObj->Exscan(local); last = first+local; } @@ -529,9 +529,9 @@ void readSolution(FILE* f, int& localnumvtx, double** solution) { } } -void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches) { +void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches, pcu::PCU *PCUObj) { fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %d \n", - PCU_Comm_Self(), numvtx, localnumvtx); + PCUObj->Self(), numvtx, localnumvtx); *matches = new apf::Gid[localnumvtx]; rewind(f); apf::Gid matchedVtx; @@ -610,9 +610,9 @@ fh = header file (there is only one for all processes), containing: ... for as many topos as are in Part 0 Repeat the above bock for each part. **/ -std::vector readHeader(std::ifstream& fh) { +std::vector readHeader(std::ifstream& fh, pcu::PCU *PCUObj) { rewindStream(fh); - const int self = PCU_Comm_Self();; + const int self = PCUObj->Self();; bool ret = seekPart(fh, std::to_string(self)); PCU_ALWAYS_ASSERT(ret); auto blockInfo = readTopoBlockInfo(fh); @@ -668,11 +668,12 @@ void readMesh(const char* meshfilename, const char* fathers2Dfilename, const char* solutionfilename, const char* connHeadfilename, - MeshInfo& mesh) { + MeshInfo& mesh, + pcu::PCU *PCUObj) { mesh.dim = 3; //FIXME - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); char filename[1024]; sprintf(filename, "%s.%d",coordfilename,self); @@ -681,9 +682,9 @@ void readMesh(const char* meshfilename, PCU_ALWAYS_ASSERT(fc); getNumVerts(fc,mesh.numVerts); mesh.localNumVerts=mesh.numVerts; - mesh.numVerts=PCU_Add_Long(mesh.numVerts); + mesh.numVerts=PCUObj->Add(mesh.numVerts); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) fprintf(stderr, "numVerts %ld\n", mesh.numVerts); readCoords(fc, mesh.localNumVerts, &(mesh.coords)); fclose(fc); @@ -715,7 +716,7 @@ void readMesh(const char* meshfilename, sprintf(filename, "%s.%d",matchfilename,self); FILE* fm = fopen(filename, "r"); PCU_ALWAYS_ASSERT(fm); - readMatches(fm, mesh.numVerts, mesh.localNumVerts, &(mesh.matches)); + readMatches(fm, mesh.numVerts, mesh.localNumVerts, &(mesh.matches), PCUObj); fclose(fm); } @@ -725,7 +726,7 @@ void readMesh(const char* meshfilename, PCU_ALWAYS_ASSERT(meshConnStream.is_open()); std::ifstream connHeadStream(connHeadfilename, std::ios::in); PCU_ALWAYS_ASSERT(connHeadStream.is_open()); - auto blockInfo = readHeader(connHeadStream); + auto blockInfo = readHeader(connHeadStream, PCUObj); connHeadStream.close(); bool rewind = true; for(auto b : blockInfo) { @@ -744,11 +745,12 @@ void readMesh(const char* meshfilename, int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); int noVerify=0; // maintain default of verifying if not explicitly requesting it off if( argc < 11 ) { - if( !PCU_Comm_Self() ) { + if( !PCUObj.get()->Self() ) { printf("Usage: %s " " " " " @@ -771,17 +773,17 @@ int main(int argc, char** argv) double t0 = pcu::Time(); MeshInfo m; - readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m); + readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m,PCUObj.get()); bool isMatched = true; if( !strcmp(argv[3], "NULL") ) isMatched = false; - if(!PCU_Comm_Self()) + if(!PCUObj.get()->Self()) fprintf(stderr, "isMatched %d\n", isMatched); gmi_model* model = gmi_load(argv[1]); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched, PCUObj.get()); apf::GlobalToVert outMap; for( size_t i=0; i< m.elements.size(); i++) { apf::assemble(mesh, m.elements[i], m.numElms[i], m.elementType[i], outMap); @@ -827,6 +829,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/measureAnisoStats.cc b/test/measureAnisoStats.cc index a3d16e968..15aa3f7c0 100644 --- a/test/measureAnisoStats.cc +++ b/test/measureAnisoStats.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -22,6 +21,7 @@ #include #include #include +#include // === includes for safe_mkdir === #include @@ -37,16 +37,17 @@ void writeTable(const char* outfile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, const char* frameName, - const char* outputPrefix); + const char* outputPrefix, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc < 5) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE1: %s " "\n", argv[0]); printf("USAGE2: %s " @@ -77,7 +78,7 @@ int main(int argc, char** argv) safe_mkdir(inPrefix); - getStats(".null", meshFile, sizeName, frameName, inPrefix); + getStats(".null", meshFile, sizeName, frameName, inPrefix, PCUObj.get()); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -88,8 +89,7 @@ int main(int argc, char** argv) SimModel_stop(); Sim_unregisterAllKeys(); #endif - - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -125,7 +125,7 @@ void writeTable(const char* outFile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, const char* frameName, - const char* outputPrefix) + const char* outputPrefix, pcu::PCU *PCUObj) { apf::Mesh2* m; @@ -133,12 +133,12 @@ void getStats( /* if it is a simmetrix mesh */ if (ph::mesh_has_ext(meshFile, "sms")) { pParMesh sim_mesh = PM_load(meshFile, NULL, NULL); - m = apf::createMesh(sim_mesh); + m = apf::createMesh(sim_mesh, PCUObj); } else #endif { // load the mesh change to desired order and write as before vtks - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile, PCUObj); } m->verify(); diff --git a/test/measureIsoStats.cc b/test/measureIsoStats.cc index 9d9e430e8..9fc6e19ba 100644 --- a/test/measureIsoStats.cc +++ b/test/measureIsoStats.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -22,6 +21,8 @@ #include #include #include +#include +#include // === includes for safe_mkdir === #include @@ -37,16 +38,17 @@ void writeTable(const char* outfile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, - const char* outputPrefix); + const char* outputPrefix, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc < 4) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE1: %s " "\n", argv[0]); printf("USAGE2: %s " @@ -76,7 +78,7 @@ int main(int argc, char** argv) safe_mkdir(inPrefix); - getStats(".null", meshFile, sizeName, inPrefix); + getStats(".null", meshFile, sizeName, inPrefix, PCUObj.get()); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -87,8 +89,7 @@ int main(int argc, char** argv) SimModel_stop(); Sim_unregisterAllKeys(); #endif - - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -124,7 +125,7 @@ void writeTable(const char* outFile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, - const char* outputPrefix) + const char* outputPrefix, pcu::PCU *PCUObj) { apf::Mesh2* m; @@ -132,12 +133,12 @@ void getStats( /* if it is a simmetrix mesh */ if (ph::mesh_has_ext(meshFile, "sms")) { pParMesh sim_mesh = PM_load(meshFile, NULL, NULL); - m = apf::createMesh(sim_mesh); + m = apf::createMesh(sim_mesh, PCUObj); } else #endif { // load the mesh change to desired order and write as before vtks - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); } m->verify(); diff --git a/test/mixedNumbering.cc b/test/mixedNumbering.cc index 6a0dbf918..8eb4f7f11 100644 --- a/test/mixedNumbering.cc +++ b/test/mixedNumbering.cc @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/test/mkmodel.cc b/test/mkmodel.cc index 3c0da2377..41bf26743 100644 --- a/test/mkmodel.cc +++ b/test/mkmodel.cc @@ -3,30 +3,31 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Create a discrete geometric model from a mesh\n" "Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); - apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], pcu_obj.get()); gmi_model* g = m->getModel(); gmi_write_dmg(g, argv[2]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/mktopomodel.cc b/test/mktopomodel.cc index af97e9f4c..522474994 100644 --- a/test/mktopomodel.cc +++ b/test/mktopomodel.cc @@ -3,18 +3,19 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Create a topological geometric model from a mesh\n" "Usage: %s \n", argv[0]); MPI_Finalize(); @@ -22,13 +23,13 @@ int main(int argc, char** argv) } gmi_register_null(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], pcu_obj.get()); apf::deriveMdsModel(m); gmi_model* g = m->getModel(); gmi_write_dmg(g, argv[2]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/modelInfo.cc b/test/modelInfo.cc index 672a1474e..b11737703 100644 --- a/test/modelInfo.cc +++ b/test/modelInfo.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -10,14 +9,18 @@ #include #endif #include //exit and exit_failure +#include +#include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 2 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -75,7 +78,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/moving.cc b/test/moving.cc index 1ee858796..36ab1d29a 100644 --- a/test/moving.cc +++ b/test/moving.cc @@ -5,9 +5,9 @@ #include #include #include -#include #include #include +#include static void writeStep(apf::Mesh* m, int i) { @@ -20,14 +20,15 @@ static void writeStep(apf::Mesh* m, int i) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 3 ) { fprintf(stderr, "Usage: %s \n", argv[0]); return 0; } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); dsp::Boundary moving; moving.insert(m->findModelEntity(2, 57)); moving.insert(m->findModelEntity(2, 62)); @@ -66,7 +67,7 @@ int main(int argc, char** argv) delete adapter; m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc index 9575e948e..6c642a448 100644 --- a/test/nedelecShapes.cc +++ b/test/nedelecShapes.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -14,6 +13,7 @@ #include #include #include +#include using namespace std; @@ -28,12 +28,13 @@ void testNedelec( int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(0); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -43,7 +44,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],PCUObj.get()); m->verify(); @@ -66,7 +67,7 @@ int main(int argc, char** argv) } apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/nektar_align.cc b/test/nektar_align.cc index 68b633203..0abb124ab 100644 --- a/test/nektar_align.cc +++ b/test/nektar_align.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -14,6 +13,7 @@ #include #endif #include +#include namespace apf { /* the more dangerous a function is, @@ -110,7 +110,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -120,7 +121,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); alignForNektar(m); m->writeNative(argv[3]); m->destroyNative(); @@ -131,6 +132,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/neper.cc b/test/neper.cc index 526ddf216..738f78d5d 100644 --- a/test/neper.cc +++ b/test/neper.cc @@ -2,28 +2,29 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2]); + apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2], pcu_obj.get()); m->verify(); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/newdim.cc b/test/newdim.cc index c1a7afece..4956a3d84 100644 --- a/test/newdim.cc +++ b/test/newdim.cc @@ -2,17 +2,18 @@ #include #include #include -#include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj.get()); apf::Vector3 points[4] = { apf::Vector3(0,0,0), apf::Vector3(1,0,0), @@ -39,7 +40,7 @@ int main(int argc, char** argv) m->verify(); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/osh2smb.cc b/test/osh2smb.cc index ed868ffe9..af0d5225f 100644 --- a/test/osh2smb.cc +++ b/test/osh2smb.cc @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include #include @@ -16,15 +16,15 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc != 4) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { std::cout << "\n"; std::cout << "usage: osh2smb in.osh in.dmg out.smb\n"; std::cout << " or: osh2smb (usage)\n"; } - PCU_Comm_Free(); MPI_Finalize(); exit(EXIT_FAILURE); } @@ -35,12 +35,12 @@ int main(int argc, char** argv) { auto lib = Omega_h::Library(&argc, &argv); Omega_h::Mesh om(&lib); Omega_h::binary::read(argv[1], lib.world(), &om); - apf::Mesh2* am = apf::makeEmptyMdsMesh(model, om.dim(), false); + apf::Mesh2* am = apf::makeEmptyMdsMesh(model, om.dim(), false, PCUObj.get()); apf::from_omega_h(am, &om); am->writeNative(argv[3]); am->destroyNative(); apf::destroyMesh(am); } - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/poisson.cc b/test/poisson.cc index 874ad03ad..df8b6421c 100644 --- a/test/poisson.cc +++ b/test/poisson.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include namespace { diff --git a/test/print_pumipic_partition.cc b/test/print_pumipic_partition.cc index 3a6be4bc8..35df92230 100644 --- a/test/print_pumipic_partition.cc +++ b/test/print_pumipic_partition.cc @@ -2,9 +2,9 @@ #include #include #include -#include #include #include +#include #ifdef HAVE_SIMMETRIX #include @@ -20,16 +20,17 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 && argc != 6) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } - if (PCU_Comm_Peers() > 1) { - if ( !PCU_Comm_Self() ) + if (pcu_obj.get()->Peers() > 1) { + if ( !pcu_obj.get()->Self() ) printf("This tool must be run in serial.\n"); MPI_Finalize(); exit(EXIT_FAILURE); @@ -44,7 +45,7 @@ int main(int argc, char** argv) gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); int num_ranks = atoi(argv[3]); //Partition the mesh (Taken from zsplit.cc) @@ -80,6 +81,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/pumi.cc b/test/pumi.cc index c407f612a..1200d862f 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -12,6 +12,7 @@ #include #include #include +#include const char* modelFile = 0; const char* meshFile = 0; @@ -87,30 +88,30 @@ int main(int argc, char** argv) //********************************************************* { MPI_Init(&argc,&argv); - //pumi_start(); - pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - pumi_printSys(PCUObj); + pumi_printSys(PCUObj.get()); #if 0 int i, processid = getpid(); - if (!PCUObj->Self()) + if (!PCUObj.get()->Self()) { - std::cout<<"Proc "<Self()<<">> pid "<Self()<<">> pid "<>i; } else - std::cout<<"Proc "<Self()<<">> pid "<Self()<<">> pid "<begin(2); gent_it!=g->end(2);++gent_it) @@ -137,16 +138,16 @@ int main(int argc, char** argv) } // load mesh per process group - PCU_ALWAYS_ASSERT(pumi_size(PCUObj)%num_in_part==0); + PCU_ALWAYS_ASSERT(pumi_size(PCUObj.get())%num_in_part==0); double begin_mem = pumi_getMem(), begin_time=pumi_getTime(); pMesh m=NULL; if (do_distr) - m = pumi_mesh_loadSerial(g, meshFile, PCUObj); + m = pumi_mesh_loadSerial(g, meshFile, PCUObj.get()); else { - m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj); // static partitioning if num_in_part=1 + m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj.get()); // static partitioning if num_in_part=1 if (num_in_part==1) pumi_mesh_write(m,"mesh.smb"); } @@ -181,7 +182,7 @@ int main(int argc, char** argv) m->end(it); pumi_mesh_distribute(m, plan); - if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] writing mesh in vtk\n"; + if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] writing mesh in vtk\n"; // write mesh in .smb pumi_mesh_write(m,outFile); @@ -195,18 +196,18 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(m); - g = pumi_geom_load(modelFile, PCUObj); - if (num_in_part==1 && pumi_size(PCUObj)>1) - m = pumi_mesh_load(g, "mesh.smb", pumi_size(PCUObj), PCUObj); + g = pumi_geom_load(modelFile, PCUObj.get()); + if (num_in_part==1 && pumi_size(PCUObj.get())>1) + m = pumi_mesh_load(g, "mesh.smb", pumi_size(PCUObj.get()), PCUObj.get()); else - m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj); - if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] delete and reload mesh\n"; + m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj.get()); + if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] delete and reload mesh\n"; pOwnership o=new testOwnership(m); pumi_ownership_verify(m, o); delete o; - if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] clean loaded tags from the mesh file\n"; + if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] clean loaded tags from the mesh file\n"; std::vector tag_vec; for (size_t n = 0; n::iterator fit=fields.begin(); fit!=fields.end(); ++fit) pumi_field_freeze(*fit); - if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] "<::iterator fit=fields.begin(); fit!=fields.end(); ++fit) pumi_field_delete(*fit); - if (!pumi_rank(PCUObj)) std::cout<<"\n[test_pumi] field and numbering deleted\n"; + if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] field and numbering deleted\n"; pumi_mesh_verify(m); TEST_MESH_TAG(m); @@ -248,9 +249,9 @@ int main(int argc, char** argv) pumi_mesh_delete(m); // print elapsed time and increased heap memory - pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem, PCUObj); + pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem, PCUObj.get()); - pumi_finalize(PCUObj); + } MPI_Finalize(); } diff --git a/test/pumiLoadMesh.cc b/test/pumiLoadMesh.cc index 5118c92ad..41e56942a 100644 --- a/test/pumiLoadMesh.cc +++ b/test/pumiLoadMesh.cc @@ -1,15 +1,17 @@ #include "mpi.h" #include "pumi.h" +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); - pGeom g = pumi_geom_load(argv[1], PCUObj, "mesh"); - pMesh m = pumi_mesh_load(g, argv[2], 1, PCUObj); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pGeom g = pumi_geom_load(argv[1], PCUObj.get(), "mesh"); + pMesh m = pumi_mesh_load(g, argv[2], 1, PCUObj.get()); pumi_mesh_delete(m); pumi_geom_delete(g); - pumi_finalize(PCUObj); + } MPI_Finalize(); } diff --git a/test/quality.cc b/test/quality.cc index 035c349a0..f5c472581 100644 --- a/test/quality.cc +++ b/test/quality.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -8,6 +7,7 @@ #include #include #include +#include namespace { @@ -19,18 +19,18 @@ double avgQuality = 0.0; double avgQualityBelowTol = 0.0; long numElemsBelowTol = 0; -static void fail(char** argv) +static void fail(char** argv, pcu::PCU *PCUObj) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } -static void getConfig(int argc, char** argv) +static void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if (argc != 4) - fail(argv); + fail(argv, PCUObj); modelFile = argv[1]; meshFile = argv[2]; qualityTol = atof(argv[3]); @@ -71,9 +71,9 @@ static void processMesh(apf::Mesh2* m) m->end(elems); } -void printDiagnostics() +void printDiagnostics(pcu::PCU *PCUObj) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) { printf("average element quality: %f\n", avgQuality); printf("quality tolerance: %f\n", qualityTol); @@ -91,15 +91,16 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + getConfig(argc,argv,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); processMesh(m); - printDiagnostics(); + printDiagnostics(PCUObj.get()); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/refine2x.cc b/test/refine2x.cc index d09c090d5..a8afcde19 100644 --- a/test/refine2x.cc +++ b/test/refine2x.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -15,6 +14,7 @@ #include #include #include +#include class AnisotropicX: public ma::AnisotropicFunction { public: @@ -85,10 +85,11 @@ class AnisotropicX: public ma::AnisotropicFunction { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc != 5) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -101,7 +102,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); AnisotropicX* ansx = new AnisotropicX(m, atoi(argv[3])); ma::Input* in = ma::makeAdvanced(ma::configure(m, ansx)); #ifdef PUMI_HAS_ZOLTAN @@ -129,6 +130,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/render.cc b/test/render.cc index 260186168..354b09ded 100644 --- a/test/render.cc +++ b/test/render.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -14,14 +13,16 @@ #endif #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -40,7 +41,7 @@ int main(int argc, char** argv) // does not fail when the input mesh is curved! crv::getBezier(2); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); std::string name = m->getShape()->getName(); int order = m->getShape()->getOrder(); @@ -59,7 +60,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/renderClass.cc b/test/renderClass.cc index 70441470b..85c1a7701 100644 --- a/test/renderClass.cc +++ b/test/renderClass.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #include #endif #include +#include static void number_dim(apf::Mesh* m, apf::FieldShape* shape, int dim, std::string const& prefix) { std::string name = prefix + "_class_dim"; @@ -34,10 +34,11 @@ static void number_dim(apf::Mesh* m, apf::FieldShape* shape, int dim, std::strin int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (!(argc == 4 || argc == 5)) { - if ( !PCU_Comm_Self() ) { + if ( !PCUObj.get()->Self() ) { printf("Usage: %s \n", argv[0]); printf(" %s \n", argv[0]); } @@ -54,7 +55,7 @@ int main(int argc, char** argv) gmi_register_mesh(); char const* modelpath = argv[1]; char const* meshpath = argv[2]; - apf::Mesh2* m = apf::loadMdsMesh(modelpath, meshpath); + apf::Mesh2* m = apf::loadMdsMesh(modelpath, meshpath, PCUObj.get()); int dim; char const* vtkpath; if (argc == 5) { @@ -75,7 +76,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/render_ascii.cc b/test/render_ascii.cc index 7682aaf69..c9e9d00ac 100644 --- a/test/render_ascii.cc +++ b/test/render_ascii.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -11,14 +10,16 @@ #include #endif #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -31,7 +32,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); apf::writeASCIIVtkFiles(argv[3], m); m->destroyNative(); apf::destroyMesh(m); @@ -41,7 +42,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/reorder.cc b/test/reorder.cc index c95c888af..e7c014fea 100644 --- a/test/reorder.cc +++ b/test/reorder.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -13,13 +12,15 @@ #include #endif #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -33,7 +34,7 @@ int main(int argc, char** argv) { #endif gmi_register_null(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); apf::MeshTag* order = Parma_BfsReorder(m); apf::reorderMdsMesh(m, order); m->writeNative(argv[3]); @@ -45,6 +46,6 @@ int main(int argc, char** argv) { SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/reposition.cc b/test/reposition.cc index 48268fb0f..326411b1d 100644 --- a/test/reposition.cc +++ b/test/reposition.cc @@ -2,19 +2,20 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #if 0 gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj.get()); apf::Vector3 vx[4] = {apf::Vector3(0,0,0), apf::Vector3(1,0,0), @@ -27,7 +28,7 @@ int main(int argc, char** argv) apf::MeshEntity* v = tv[0]; #else gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* v; while ((v = m->iterate(it))) { @@ -43,6 +44,6 @@ int main(int argc, char** argv) apf::writeVtkFiles("before", m); ma::repositionVertex(m, v, 20, 1.0); apf::writeVtkFiles("after", m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc index 085966460..d4c662056 100644 --- a/test/residualErrorEstimation_test.cc +++ b/test/residualErrorEstimation_test.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #include #endif #include +#include void E_exact(const apf::Vector3 &x, apf::Vector3& E); double computeElementExactError(apf::Mesh* mesh, apf::MeshEntity* e, @@ -27,7 +27,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -37,7 +38,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); kappa = freq * M_PI; @@ -96,7 +97,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/rm_extrusion.cc b/test/rm_extrusion.cc index 1f98a3e01..c4c805b0e 100644 --- a/test/rm_extrusion.cc +++ b/test/rm_extrusion.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include @@ -36,7 +36,7 @@ void M_removeSurfaceExtrusionConstraints(pMesh, pPList); #endif -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { opterr = 0; @@ -63,18 +63,18 @@ void getConfig(int argc, char** argv) { gmi_native_path = optarg; break; case '?': - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf ("warning: skipping unrecognized option\n"); break; default: - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -83,7 +83,7 @@ void getConfig(int argc, char** argv) { sms_path = argv[i++]; smsNew_path = argv[i++]; - if (!PCU_Comm_Self()) { + if (!PCUObj->Self()) { printf ("enable_log %d\n", should_log); printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", gmi_native_path, gmi_path, sms_path, smsNew_path); @@ -93,7 +93,8 @@ void getConfig(int argc, char** argv) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); @@ -101,7 +102,7 @@ int main(int argc, char** argv) Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv); + getConfig(argc, argv, PCUObj.get()); if( should_log ) Sim_logOn("rm_extrusion.sim.log"); @@ -116,21 +117,21 @@ int main(int argc, char** argv) else mdl = gmi_load(gmi_path); pGModel simModel = gmi_export_sim(mdl); - if(!PCU_Comm_Self()) + if(!PCUObj.get()->Self()) fprintf(stderr, "Read model\n"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); pMesh sim_mesh = M_load(sms_path, simModel, progress); - if(!PCU_Comm_Self()) + if(!PCUObj.get()->Self()) fprintf(stderr, "Read mesh\n"); M_removeSurfaceExtrusionConstraints(sim_mesh, NULL); - if(!PCU_Comm_Self()) + if(!PCUObj.get()->Self()) fprintf(stderr, "Removed surface extrusion constraints\n"); M_write(sim_mesh, smsNew_path, 0, progress); double t1 = pcu::Time(); - if(!PCU_Comm_Self()) + if(!PCUObj.get()->Self()) fprintf(stderr, "read the mesh, removed the face extrusion attributes, and wrote the mesh %f seconds\n", t1-t0); M_release(sim_mesh); @@ -144,6 +145,6 @@ int main(int argc, char** argv) MS_exit(); if( should_log ) Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc index 1d5b9ae3f..81191cad4 100644 --- a/test/runSimxAnisoAdapt.cc +++ b/test/runSimxAnisoAdapt.cc @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -72,18 +72,19 @@ void addFields(apf::Mesh2* m, const vector &adaptedFrames); apf::Mesh2* convertToPumi( pMesh simxMesh, int dim, - const char* sizeName, const char* frameName); + const char* sizeName, const char* frameName, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); MS_init(); // Call before calling Sim_readLicenseFile Sim_readLicenseFile(0); SimDiscrete_start(0); // initialize GeomSim Discrete library if (argc < 7) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE: %s " " \n", argv[0]); } @@ -94,7 +95,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_register_null(); - PCU_ALWAYS_ASSERT_VERBOSE(PCU_Comm_Peers() == 1, + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.get()->Peers() == 1, "This utility only works for serial meshes!"); const char* inputPumiModel = argv[1]; @@ -118,7 +119,7 @@ int main(int argc, char** argv) sprintf(outImprovedSimxMesh, "%s_adapted_improved.sms", prefix); sprintf(outImprovedPumiMesh, "%s_adapted_improved.smb", prefix); - apf::Mesh2* m = apf::loadMdsMesh(inputPumiModel, inputPumiMesh); + apf::Mesh2* m = apf::loadMdsMesh(inputPumiModel, inputPumiMesh, PCUObj.get()); char message[512]; // first find the sizes field @@ -193,7 +194,7 @@ int main(int argc, char** argv) printf("%s\n", outAdaptedSimxMesh); printf("%s\n", outAdaptedPumiMesh); M_write(simxMesh,outAdaptedSimxMesh, 0,0); // write out the initial mesh data - apf::Mesh2* m2 = convertToPumi(simxMesh, dim, sizeName, frameName); + apf::Mesh2* m2 = convertToPumi(simxMesh, dim, sizeName, frameName, PCUObj.get()); m2->writeNative(outAdaptedPumiMesh); printf("===DONE===\n"); @@ -207,7 +208,7 @@ int main(int argc, char** argv) printf("%s\n", outImprovedSimxMesh); printf("%s\n", outImprovedPumiMesh); M_write(simxMesh,outImprovedSimxMesh, 0,0); // write out the initial mesh data - apf::Mesh2* m3 = convertToPumi(simxMesh, dim, sizeName, frameName); + apf::Mesh2* m3 = convertToPumi(simxMesh, dim, sizeName, frameName, PCUObj.get()); m3->writeNative(outImprovedPumiMesh); printf("===DONE===\n"); @@ -228,7 +229,7 @@ int main(int argc, char** argv) SimDiscrete_stop(0); Sim_unregisterAllKeys(); MS_exit(); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -585,7 +586,8 @@ void addFields(apf::Mesh2* m, apf::Mesh2* convertToPumi( pMesh simxMesh, int dim, const char* sizeName, - const char* frameName) + const char* frameName, + pcu::PCU *PCUObj) { double* adaptedCoords; apf::Gid* adaptedConns; @@ -596,7 +598,7 @@ apf::Mesh2* convertToPumi( adaptedNumVerts, adaptedNumElems, adaptedSizes, adaptedFrames); gmi_model* nullModel = gmi_load(".null"); - apf::Mesh2* m2 = apf::makeEmptyMdsMesh(nullModel, dim, false); + apf::Mesh2* m2 = apf::makeEmptyMdsMesh(nullModel, dim, false, PCUObj); apf::GlobalToVert outMap; apf::construct(m2, adaptedConns, adaptedNumElems, apf::Mesh::TET, outMap);; apf::alignMdsRemotes(m2); diff --git a/test/scale.cc b/test/scale.cc index 3816dc940..40f153035 100644 --- a/test/scale.cc +++ b/test/scale.cc @@ -3,9 +3,9 @@ #include #include #include -#include #include #include +#include namespace { @@ -16,8 +16,8 @@ struct Scale { double s; }; -static void print_usage(char** argv) { - if (! PCU_Comm_Self()) +static void print_usage(char** argv, pcu::PCU *PCUObj) { + if (! PCUObj->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -55,10 +55,11 @@ static void scale_mesh(apf::Mesh2* m, Scale const& s) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - if (argc != 8) print_usage(argv); + if (argc != 8) print_usage(argv, PCUObj.get()); const char* gfile = argv[1]; const char* mfile = argv[2]; const char* ofile = argv[3]; @@ -67,13 +68,13 @@ int main(int argc, char** argv) { scale.y = atof(argv[5]); scale.z = atof(argv[6]); scale.s = atof(argv[7]); - apf::Mesh2* m = apf::loadMdsMesh(gfile, mfile); + apf::Mesh2* m = apf::loadMdsMesh(gfile, mfile, PCUObj.get()); m->verify(); scale_mesh(m, scale); m->verify(); m->writeNative(ofile); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/serialize.cc b/test/serialize.cc index 3a40c2663..2ba122835 100644 --- a/test/serialize.cc +++ b/test/serialize.cc @@ -18,7 +18,7 @@ struct GroupCode : public Parma_GroupCode { apf::Mesh2* mesh; const char* meshFile; - void run(int, pcu::PCU*) + void run(int) { mesh->writeNative(meshFile); } diff --git a/test/shapefun.cc b/test/shapefun.cc index 00aaa7036..bbf7fd336 100644 --- a/test/shapefun.cc +++ b/test/shapefun.cc @@ -3,9 +3,9 @@ #include #include #include -#include #include #include +#include #include @@ -206,10 +206,10 @@ void testQuadrilateralNodeValues() { testNodeValues(shp2, nodes2, 9); } -void testVolume(int type, apf::Vector3 const* points, double volume) +void testVolume(int type, apf::Vector3 const* points, double volume, pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* e = apf::buildOneElement( m, m->findModelEntity(3,0), type, points); double v = apf::measure(m,e); @@ -218,7 +218,7 @@ void testVolume(int type, apf::Vector3 const* points, double volume) apf::destroyMesh(m); } -void testPrismVolume() +void testPrismVolume(pcu::PCU *PCUObj) { apf::Vector3 points[6] = {apf::Vector3(0,0,0), @@ -227,10 +227,10 @@ void testPrismVolume() apf::Vector3(0,0,1), apf::Vector3(1,0,1), apf::Vector3(0,1,1)}; - testVolume(apf::Mesh::PRISM, points, 0.5); + testVolume(apf::Mesh::PRISM, points, 0.5, PCUObj); } -void testPyramidVolume() +void testPyramidVolume(pcu::PCU *PCUObj) { apf::Vector3 points[5] = {apf::Vector3(0,0,0), @@ -238,13 +238,14 @@ void testPyramidVolume() apf::Vector3(1,1,0), apf::Vector3(0,1,0), apf::Vector3(0,0,3)}; - testVolume(apf::Mesh::PYRAMID, points, 1); + testVolume(apf::Mesh::PYRAMID, points, 1, PCUObj); } int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); testP1LineNodeValues(); @@ -259,8 +260,8 @@ int main(int argc, char** argv) testPrismNodeValues(); testPyramidNodeValues(); testQuadrilateralNodeValues(); - testPrismVolume(); - testPyramidVolume(); - PCU_Comm_Free(); + testPrismVolume(PCUObj.get()); + testPyramidVolume(PCUObj.get()); + } MPI_Finalize(); } diff --git a/test/shapefun2.cc b/test/shapefun2.cc index 121b07f58..9fde54b5d 100644 --- a/test/shapefun2.cc +++ b/test/shapefun2.cc @@ -4,9 +4,9 @@ #include #include #include -#include #include #include +#include namespace test { @@ -131,7 +131,7 @@ static void checkEntityShape(apf::Mesh* lm, apf::Mesh* m, /* For every shape function we test, for every type we test, * create two meshes */ -static void checkFieldShape(apf::FieldShape* fs) +static void checkFieldShape(apf::FieldShape* fs, pcu::PCU *PCUObj) { for (int type = 1; type < apf::Mesh::TYPES; ++type){ apf::EntityShape* es = fs->getEntityShape(type); @@ -141,8 +141,8 @@ static void checkFieldShape(apf::FieldShape* fs) gmi_model* lmodel = gmi_load(".null"); gmi_model* model = gmi_load(".null"); - apf::Mesh2* lm = apf::makeEmptyMdsMesh(lmodel, typeDim, false); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, typeDim, false); + apf::Mesh2* lm = apf::makeEmptyMdsMesh(lmodel, typeDim, false, PCUObj); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, typeDim, false, PCUObj); apf::MeshEntity* le = apf::buildOneElement( lm, lm->findModelEntity(typeDim,0), type, points[type]); @@ -171,7 +171,8 @@ static void checkFieldShape(apf::FieldShape* fs) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); @@ -180,9 +181,9 @@ int main(int argc, char** argv) crv::getBezier(1),crv::getBezier(2)}; for (int i = 0; i < 4; ++i) - test::checkFieldShape(fs[i]); + test::checkFieldShape(fs[i], PCUObj.get()); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/simZBalance.cc b/test/simZBalance.cc index 9494e4342..7676d487b 100644 --- a/test/simZBalance.cc +++ b/test/simZBalance.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -11,26 +10,8 @@ #include #include #include +#include -static void initialize(int argc, char** argv) { - MPI_Init(&argc, &argv); - PCU_Comm_Init(); - lion_set_verbosity(1); - Sim_readLicenseFile(NULL); - gmi_sim_start(); - gmi_register_sim(); - SimPartitionedMesh_start(&argc, &argv); - MS_init(); -} - -static void finalize() { - MS_exit(); - SimPartitionedMesh_stop(); - gmi_sim_stop(); - Sim_unregisterAllKeys(); - PCU_Comm_Free(); - MPI_Finalize(); -} static void load_balance(apf::Mesh2* m) { Parma_PrintPtnStats(m, "initial"); @@ -44,18 +25,33 @@ static void load_balance(apf::Mesh2* m) { } int main(int argc, char** argv) { - initialize(argc, argv); + MPI_Init(&argc, &argv); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + lion_set_verbosity(1); + Sim_readLicenseFile(NULL); + gmi_sim_start(); + gmi_register_sim(); + SimPartitionedMesh_start(&argc, &argv); + MS_init(); + const char* smd_file = argv[1]; const char* sms_file = argv[2]; gmi_model* apf_model = gmi_sim_load(0, smd_file); pGModel sim_model = gmi_export_sim(apf_model); pParMesh sim_mesh = PM_load(sms_file, sim_model, NULL); - apf::Mesh2* apf_mesh = apf::createMesh(sim_mesh); + apf::Mesh2* apf_mesh = apf::createMesh(sim_mesh, PCUObj.get()); //apf_mesh->verify(); <- this calls Simmetrix's verify function apf::verify(apf_mesh); load_balance(apf_mesh); apf_mesh->destroyNative(); apf::destroyMesh(apf_mesh); gmi_destroy(apf_model); - finalize(); + + MS_exit(); + SimPartitionedMesh_stop(); + gmi_sim_stop(); + Sim_unregisterAllKeys(); + } + MPI_Finalize(); } diff --git a/test/sim_part.cc b/test/sim_part.cc index 13d4427f0..811f8abd4 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -22,7 +22,6 @@ // /* cheap hackish way to get SIM_PARASOLID and SIM_ACIS */ -#include #include #include "gmi_sim_config.h" #include @@ -45,6 +44,7 @@ #include #include #include +#include using namespace std; @@ -66,7 +66,8 @@ void messageHandler(int type, const char *msg); int main(int argc, char **argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); pcu::Protect(); // Initialize PartitionedMesh - this should be the first Simmetrix call @@ -106,7 +107,7 @@ int main(int argc, char **argv) outmeshFilename = tmp.c_str(); /* print message */ - if (PCU_Comm_Self()==0) { + if (PCUObj.get()->Self()==0) { cout<getPCU()->Self()==0) { + if (PCUObj.get()->Self()==0) { cout<<"**********************************"< #include #include -#include #include #include #include +#include #include @@ -16,21 +16,21 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc != 4) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { std::cout << "\n"; std::cout << "usage: smb2osh in.dmg in.smb out.osh\n"; std::cout << " or: smb2osh (usage)\n"; } - PCU_Comm_Free(); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); gmi_register_null(); - apf::Mesh2* am = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* am = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); { auto lib = Omega_h::Library(&argc, &argv); Omega_h::Mesh om(&lib); @@ -39,6 +39,6 @@ int main(int argc, char** argv) { apf::destroyMesh(am); Omega_h::binary::write(argv[3], &om); } - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/snap.cc b/test/snap.cc index 6134d245d..825645c46 100644 --- a/test/snap.cc +++ b/test/snap.cc @@ -2,25 +2,26 @@ #include #include #include -#include #include #include #include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); const ma::Input* in = ma::configureIdentity(m); ma::adapt(in); m->writeNative(argv[3]); @@ -30,7 +31,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/swapDoubles.cc b/test/swapDoubles.cc index daba9467c..7db5e5966 100644 --- a/test/swapDoubles.cc +++ b/test/swapDoubles.cc @@ -1,12 +1,15 @@ -#include +//#include +#include #include //pcu_swap_doubles #include //PCU_ALWAYS_ASSERT #include //iota #include //cerr +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); const size_t n = 2; double *d_orig = new double[n]; std::iota(d_orig,d_orig+n,0); @@ -25,7 +28,7 @@ int main(int argc, char** argv) { } delete [] d_orig; delete [] d; - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/test_integrator.cc b/test/test_integrator.cc index 682ea2eee..ee715ace5 100644 --- a/test/test_integrator.cc +++ b/test/test_integrator.cc @@ -2,11 +2,11 @@ #include #include #include -#include #include #include #include #include +#include class CountIntegrator : public apf::Integrator { protected: @@ -24,13 +24,14 @@ class CountIntegrator : public apf::Integrator { }; int main(int argc, char ** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); // argument should be model, mesh PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); gmi_register_null(); - apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); CountIntegrator * countInt = new CountIntegrator(); // test integration over implicitly defined mesh dimension countInt->process(mesh); @@ -47,7 +48,7 @@ int main(int argc, char ** argv) { delete countInt; mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/test_matrix_grad.cc b/test/test_matrix_grad.cc index 71eb13729..cc9bbf805 100644 --- a/test/test_matrix_grad.cc +++ b/test/test_matrix_grad.cc @@ -2,12 +2,12 @@ #include #include #include -#include #include #include #include #include #include +#include /* * This test sets a nodal point matrix on a linear mesh. It then computes the gradient @@ -94,10 +94,11 @@ int main(int argc, char* argv[]) return 1; } MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); int order=1; apf::Field* nodal_matrix_fld = apf::createLagrangeField(mesh, "matrix", apf::MATRIX, order); apf::Field* matrix_deriv = apf::createPackedField(mesh, "matrix_deriv", 27, apf::getIPShape(3,order)); @@ -116,7 +117,7 @@ int main(int argc, char* argv[]) std::cout<<"Done"<destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/test_scaling.cc b/test/test_scaling.cc index 0a18cddea..da18142b0 100644 --- a/test/test_scaling.cc +++ b/test/test_scaling.cc @@ -5,19 +5,20 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); apf::Field* identity_size = samSz::isoSize(m); double scaling_factor = sam::getIsoLengthScalar(identity_size, m->count(m->getDimension())); @@ -26,6 +27,6 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(0.5 < scaling_factor); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/test_verify.cc b/test/test_verify.cc index fdc57e0f1..75e4d7b13 100644 --- a/test/test_verify.cc +++ b/test/test_verify.cc @@ -3,20 +3,21 @@ #include #include #include -#include #include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); apf::DynamicArray fields(10); apf::FieldShape* shapes[10] = { apf::getLagrange(1), @@ -43,6 +44,6 @@ int main(int argc, char** argv) apf::destroyField(fields[i]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/tetrahedronize.cc b/test/tetrahedronize.cc index edd035d3f..b1375de6b 100644 --- a/test/tetrahedronize.cc +++ b/test/tetrahedronize.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -11,12 +10,14 @@ #include #endif #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +27,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldTurnLayerToTets = true; ma::adapt(in); @@ -39,7 +40,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/torus_ma_test.cc b/test/torus_ma_test.cc index d7d6bcc06..137ffbfe5 100644 --- a/test/torus_ma_test.cc +++ b/test/torus_ma_test.cc @@ -3,9 +3,9 @@ #include #include #include -#include #include #include +#include class CylindricalShock : public ma::AnisotropicFunction { @@ -48,10 +48,11 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); apf::writeVtkFiles("torus_before",m); CylindricalShock sf(m); @@ -69,7 +70,7 @@ int main(int argc, char** argv) apf::writeVtkFiles("torus_after",m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ugridptnstats.cc b/test/ugridptnstats.cc index d974131d4..0fffb75b9 100644 --- a/test/ugridptnstats.cc +++ b/test/ugridptnstats.cc @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include const double vtxw = 1.0; const double edgew = 1.0; @@ -20,7 +20,8 @@ const double weights[8] = {vtxw, edgew, triw, quadw, tetw, hexw, przw, pyrw}; int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); PCU_ALWAYS_ASSERT( 3 == argc ); @@ -29,6 +30,6 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(".null"); apf::printUgridPtnStats(g,ugridfile,ptnfile,weights); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/uniform.cc b/test/uniform.cc index 427598a06..f18b211cb 100644 --- a/test/uniform.cc +++ b/test/uniform.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -12,15 +11,16 @@ #endif #include #include +#include const char* modelFile = 0; const char* meshFile = 0; const char* outFile = 0; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -34,7 +34,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -44,8 +45,8 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 1)); if (in->shouldSnap) { in->shouldSnap = false; @@ -62,7 +63,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/verify_2nd_order_shapes.cc b/test/verify_2nd_order_shapes.cc index bc26005eb..b4143ed07 100644 --- a/test/verify_2nd_order_shapes.cc +++ b/test/verify_2nd_order_shapes.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -15,13 +14,15 @@ #endif #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 2 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -36,7 +37,7 @@ int main(int argc, char** argv) { gmi_register_null(); gmi_register_mesh(); gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[1],PCUObj.get()); int dim = m->getDimension(); @@ -110,6 +111,6 @@ int main(int argc, char** argv) { SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } From 1eebc1e620171cc65386ed5e056a75929bb1aa29 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 26 Apr 2024 12:06:36 -0400 Subject: [PATCH 107/141] Mesh interface fixes --- apf/apfMesh2.cc | 88 ++++++++++++++++++++++++++++++++++++----------- apf/apfMesh2.h | 2 +- mds/apfBox.cc | 2 +- mds/apfMDS.cc | 15 ++++---- mds/apfMDS.h | 8 ++--- mds/mdsGmsh.cc | 14 ++++++-- mds/mdsUgrid.cc | 18 +++++++--- pumi/pumi_mesh.cc | 4 ++- 8 files changed, 111 insertions(+), 40 deletions(-) diff --git a/apf/apfMesh2.cc b/apf/apfMesh2.cc index cd87fa9e0..329867b63 100644 --- a/apf/apfMesh2.cc +++ b/apf/apfMesh2.cc @@ -203,17 +203,29 @@ void stitchMesh(Mesh2* m) } } -static void packTagClone(Mesh2* m, MeshTag* t, int to) +static void packTagClone(Mesh2* m, MeshTag* t, int to, pcu::PCU *PCUObj) { std::string name; name = m->getTagName(t); - packString(name, to, m->getPCU()); + if(PCUObj != nullptr){ + packString(name, to, PCUObj); + } else { + packString(name, to, m->getPCU()); + } int type; type = m->getTagType(t); - m->getPCU()->Pack(to, type); + if(PCUObj != nullptr){ + PCUObj->Pack(to, type); + } else { + m->getPCU()->Pack(to, type); + } int size; size = m->getTagSize(t); - m->getPCU()->Pack(to, size); + if(PCUObj != nullptr){ + PCUObj->Pack(to, size); + } else { + m->getPCU()->Pack(to, size); + } } static MeshTag* unpackTagClone(Mesh2* m) @@ -232,17 +244,22 @@ static MeshTag* unpackTagClone(Mesh2* m) return 0; } -static void packTagClones(Mesh2* m, int to) +static void packTagClones(Mesh2* m, int to, pcu::PCU *PCUObj) { DynamicArray tags; m->getTags(tags); int n = tags.getSize(); - m->getPCU()->Pack(to, n); + if(PCUObj != nullptr){ + PCUObj->Pack(to, n); + } else { + m->getPCU()->Pack(to, n); + } + /* warning! this loop goes backward to cater to MDS implementation-specific behavior. please forgive me. */ for (int i = n - 1; i >= 0; --i) - packTagClone(m, tags[i], to); + packTagClone(m, tags[i], to, PCUObj); } static void unpackTagClones(Mesh2* m) @@ -253,16 +270,36 @@ static void unpackTagClones(Mesh2* m) unpackTagClone(m); } -static void packFieldClone(Field* f, int to) +static void packFieldClone(Field* f, int to, pcu::PCU *PCUObj) { std::string name = f->getName(); - packString(name, to, f->getMesh()->getPCU()); + if(PCUObj != nullptr){ + packString(name, to, PCUObj); + } else { + packString(name, to, f->getMesh()->getPCU()); + } + int valueType = f->getValueType(); - f->getMesh()->getPCU()->Pack(to, valueType); + if(PCUObj != nullptr){ + PCUObj->Pack(to, valueType); + } else { + f->getMesh()->getPCU()->Pack(to, valueType); + } + int components = f->countComponents(); - f->getMesh()->getPCU()->Pack(to, components); + if(PCUObj != nullptr){ + PCUObj->Pack(to, components); + } else { + f->getMesh()->getPCU()->Pack(to, components); + } + std::string shapeName = f->getShape()->getName(); - packString(shapeName, to, f->getMesh()->getPCU()); + if(PCUObj != nullptr){ + packString(shapeName, to, PCUObj); + } else { + packString(shapeName, to, f->getMesh()->getPCU()); + } + /* warning! this only supports tag-stored fields */ } @@ -280,12 +317,17 @@ static Field* unpackFieldClone(Mesh2* m) return makeField(m, name.c_str(), valueType, components, shape, new TagDataOf); } -static void packFieldClones(Mesh2* m, int to) +static void packFieldClones(Mesh2* m, int to, pcu::PCU *PCUObj) { int n = m->countFields(); - m->getPCU()->Pack(to, n); + if(PCUObj != nullptr){ + PCUObj->Pack(to, n); + } else { + m->getPCU()->Pack(to, n); + } + for (int i = 0; i < n; ++i) - packFieldClone(m->getField(i), to); + packFieldClone(m->getField(i), to, PCUObj); } static void unpackFieldClones(Mesh2* m) @@ -296,10 +338,14 @@ static void unpackFieldClones(Mesh2* m) unpackFieldClone(m); } -static void packMeshShape(Mesh2* m, int to) +static void packMeshShape(Mesh2* m, int to, pcu::PCU *PCUObj) { std::string shapeName = m->getShape()->getName(); - packString(shapeName, to, m->getPCU()); + if(PCUObj != nullptr){ + packString(shapeName, to, PCUObj); + } else { + packString(shapeName, to, m->getPCU()); + } } static void unpackMeshShape(Mesh2* m) @@ -312,11 +358,11 @@ static void unpackMeshShape(Mesh2* m) } } -void packDataClone(Mesh2* m, int to) +void packDataClone(Mesh2* m, int to, pcu::PCU *PCUObj) { - packTagClones(m, to); - packMeshShape(m, to); - packFieldClones(m, to); + packTagClones(m, to, PCUObj); + packMeshShape(m, to, PCUObj); + packFieldClones(m, to, PCUObj); } void unpackDataClone(Mesh2* m) diff --git a/apf/apfMesh2.h b/apf/apfMesh2.h index 2ade4062f..c11c0a534 100644 --- a/apf/apfMesh2.h +++ b/apf/apfMesh2.h @@ -199,7 +199,7 @@ void stitchMesh(Mesh2* m); /** \brief removes all entities and fields. */ void clear(Mesh2* m); -void packDataClone(Mesh2* m, int to); +void packDataClone(Mesh2* m, int to, pcu::PCU *PCUObj = nullptr); void unpackDataClone(Mesh2* m); // common functions for migration/ghosting/distribution diff --git a/mds/apfBox.cc b/mds/apfBox.cc index a8b281009..37fe0f2b6 100644 --- a/mds/apfBox.cc +++ b/mds/apfBox.cc @@ -78,7 +78,7 @@ BoxBuilder::BoxBuilder(int nx, int ny, int nz, is_simplex = is; formModelTable(); gmi_model* gm = buildModel(); - m = makeEmptyMdsMesh(gm, dim, false); + m = makeEmptyMdsMesh(gm, dim, false, PCUObj); v.resize(grid.total()); buildMeshAndModel(); } diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index da696372b..53369737f 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -940,9 +940,14 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder, bool copy_data) return new MeshMDS(model, from, &(node_arr[0]), &(elem_arr[0]), copy_data); } -Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile) +Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { - Mesh2* m = new MeshMDS(model, meshfile); + Mesh2* m; + if(PCUObj != nullptr){ + m = new MeshMDS(model, meshfile, PCUObj); + } else { + m = new MeshMDS(model, meshfile); + } return m; } @@ -1023,7 +1028,6 @@ Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expan { double t0 = pcu::Time(); int self = expandedPCU->Self(); - //PCU_ALWAYS_ASSERT_VERBOSE(self == expandedPCU->Self(), "pcu_comm_self doesn't equal pcuobj->self\n"); int outputPartCount = expandedPCU->Peers(); apf::Expand expand(inputPartCount, outputPartCount); apf::Contract contract(inputPartCount, outputPartCount); @@ -1038,7 +1042,7 @@ Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expan for (int i = self + 1; i < outputPartCount && !contract.isValid(i); ++i) { expandedPCU->Pack(i, dim); expandedPCU->Pack(i, isMatched); - packDataClone(m, i); + packDataClone(m, i, expandedPCU); } } expandedPCU->Send(); @@ -1113,9 +1117,8 @@ Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, } Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, - int factor, pcu::PCU *PCUObj) + int factor, pcu::PCU *PCUObj, pcu::PCU *oldPCU) { - //PCU_ALWAYS_ASSERT_VERBOSE(PCU_Comm_Self() == PCUObj->Self(), "repeartMdsMesh assert failed\n"); m = expandMdsMesh(m, g, PCUObj->Peers() / factor, PCUObj); double t0 = pcu::Time(); if (PCUObj->Self() % factor != 0) diff --git a/mds/apfMDS.h b/mds/apfMDS.h index 4a63dd328..b0557ce43 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -91,7 +91,7 @@ Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile); // make a serial mesh on all processes - no pmodel & remote link setup -Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile); +Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj = nullptr); /** \brief create an MDS mesh from an existing mesh \param from the mesh to copy @@ -117,7 +117,7 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder=false, bool copy void reorderMdsMesh(Mesh2* mesh, MeshTag* t = 0); Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor); -Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj); +Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj, pcu::PCU *oldPCU = nullptr); Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount); Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU); @@ -222,10 +222,10 @@ Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj = nu Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename, pcu::PCU *PCUObj = nullptr); -Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename); +Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj = nullptr); void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile, - const double elmWeights[]); + const double elmWeights[], pcu::PCU *PCUObj = nullptr); /** \brief load an MDS mesh from ANSYS .node and .elem files \details this call takes two filenames, one diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc index b6d3f8524..99ca5af24 100644 --- a/mds/mdsGmsh.cc +++ b/mds/mdsGmsh.cc @@ -461,7 +461,12 @@ void gmshFindDmg(const char* fnameDmg, const char* filename) Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); + Mesh2* m; + if(PCUObj != nullptr){ + m = makeEmptyMdsMesh(g, 0, false, PCUObj); + } else { + m = makeEmptyMdsMesh(g, 0, false); + } readGmsh(m, filename); return m; } @@ -469,7 +474,12 @@ Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename, pcu::PCU *PCUObj) { gmshFindDmg(fnameDmg, filename); // new function that scans $Entities and writes a dmg - Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); + Mesh2* m; + if(PCUObj != nullptr){ + m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); + } else { + m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false); + } readGmsh(m, filename); return m; } diff --git a/mds/mdsUgrid.cc b/mds/mdsUgrid.cc index ceb8a001b..f7fe31545 100644 --- a/mds/mdsUgrid.cc +++ b/mds/mdsUgrid.cc @@ -661,9 +661,14 @@ namespace { } namespace apf { - Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename) + Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false); + Mesh2* m; + if(PCUObj != nullptr){ + m = makeEmptyMdsMesh(g, 0, false, PCUObj); + } else { + m = makeEmptyMdsMesh(g, 0, false); + } header hdr; Reader r; initReader(&r, m, filename); @@ -684,8 +689,13 @@ namespace apf { } void printUgridPtnStats(gmi_model* g, const char* ufile, const char* vtxptn, - const double elmWeights[]) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false); + const double elmWeights[], pcu::PCU *PCUObj) { + Mesh2* m; + if(PCUObj != nullptr){ + m = makeEmptyMdsMesh(g, 0, false, PCUObj); + } else { + m = makeEmptyMdsMesh(g, 0, false); + } apf::changeMdsDimension(m, 3); printPtnStats(m, ufile, vtxptn, elmWeights); m->destroyNative(); diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index f04633638..5b6540da9 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -188,6 +188,7 @@ void split_comm(int num_out_comm, pcu::PCU &PCUObj) PCUObj.SwitchMPIComm(groupComm); } + void merge_comm(MPI_Comm oldComm, pcu::PCU &PCUObj) { MPI_Comm prevComm = PCUObj.GetMPIComm(); @@ -196,6 +197,7 @@ void merge_comm(MPI_Comm oldComm, pcu::PCU &PCUObj) } + pGeom pumi_mesh_getGeom(pMesh) { return pumi::instance()->model; @@ -375,7 +377,7 @@ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool st int num_target_part = PCUObj->Peers(); split_comm(num_target_part, *PCUObj); // no pmodel & remote links setup - pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename); + pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename, PCUObj); merge_comm(prevComm, *PCUObj); if (!PCUObj->Self()) lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); From 94dcc926a893b72d058fe83fea16e7e1bfe11f96 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 26 Apr 2024 12:07:12 -0400 Subject: [PATCH 108/141] Test files changed to use new PCU --- test/makeAllCavities.cc | 2 +- test/ptnParma.cc | 48 ++++++++++++++++--------------------- test/split.cc | 40 +++++++++++++------------------ test/ugrid.cc | 33 +++++++++++-------------- test/verify.cc | 9 +++---- test/verify_convert.cc | 22 +++++++++-------- test/visualizeAnisoSizes.cc | 21 +++++++++------- test/viz.cc | 9 +++---- test/vtxBalance.cc | 11 +++++---- test/vtxEdgeElmBalance.cc | 11 +++++---- test/vtxElmBalance.cc | 11 +++++---- test/vtxElmMixedBalance.cc | 9 +++---- test/writeIPFieldTest.cc | 9 +++---- test/writePart.cc | 11 +++++---- test/writeVtxPtn.cc | 15 ++++++------ test/xgc_split.cc | 14 ++++++----- test/zbalance.cc | 11 +++++---- test/zsplit.cc | 40 +++++++++++++------------------ 18 files changed, 160 insertions(+), 166 deletions(-) diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc index 6328a7ddc..eedf4155f 100644 --- a/test/makeAllCavities.cc +++ b/test/makeAllCavities.cc @@ -293,7 +293,7 @@ static apf::Mesh2* makePoint( apf::MeshEntity* e) { PCU_ALWAYS_ASSERT(m->getType(e) == apf::Mesh::VERTEX); - apf::Mesh2* sphMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + apf::Mesh2* sphMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, m->getPCU()); double xrange[2] = {1.e16, -1.e16}; double yrange[2] = {1.e16, -1.e16}; double zrange[2] = {1.e16, -1.e16}; diff --git a/test/ptnParma.cc b/test/ptnParma.cc index 765bf9bfc..701143907 100644 --- a/test/ptnParma.cc +++ b/test/ptnParma.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -15,6 +14,7 @@ #include #endif #include +#include namespace { @@ -85,22 +85,14 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -void switchToMasters() +pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); -} - -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); + return new pcu::PCU(groupComm); } void runParma(apf::Mesh2* m) { @@ -124,28 +116,29 @@ void runParma(apf::Mesh2* m) { Parma_PrintPtnStats(m, "final"); } -void mymain(bool ismaster) +void mymain(bool ismaster, pcu::PCU *PCUObj) { gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = NULL; apf::Migration* plan = NULL; - switchToMasters(); + pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj) if (ismaster) { - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile,groupedPCUObj); Parma_PrintPtnStats(m, "initial"); plan = getPlan(m); } - switchToAll(); - m = apf::repeatMdsMesh(m, g, plan, partitionFactor); + delete groupedPCUObj; + if(m != nullptr) m->switchPCU(PCUObj); + m = apf::repeatMdsMesh(m, g, plan, partitionFactor, PCUObj); runParma(m); m->writeNative(outFile); freeMesh(m); } -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 8 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s " " <0:global|1:local>\n", argv[0]); MPI_Finalize(); @@ -158,7 +151,7 @@ void getConfig(int argc, char** argv) method = argv[5]; approach = argv[6]; isLocal = atoi(argv[7]); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) lion_eprint(1, "INPUTS model %s mesh %s out %s factor %d " "method %s approach %s isLocal %d\n", modelFile, meshFile, outFile, partitionFactor, method, approach, isLocal); @@ -169,9 +162,10 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - if( !PCU_Comm_Self() ) + if( !PCUObj.get()->Self() ) lion_oprint(1, "PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); #ifdef HAVE_SIMMETRIX MS_init(); @@ -182,17 +176,17 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); lion_set_verbosity(1); - getConfig(argc,argv); - if (PCU_Comm_Self() % partitionFactor) - mymain(false); + getConfig(argc,argv,PCUObj.get()); + if (PCUObj.get()->Self() % partitionFactor) + mymain(false, PCUObj.get()); else - mymain(true); + mymain(true, PCUObj.get()); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/split.cc b/test/split.cc index e56b5dc7c..8855485dc 100644 --- a/test/split.cc +++ b/test/split.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -13,6 +12,7 @@ #endif #include #include +#include namespace { @@ -38,28 +38,20 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -void switchToOriginals() +pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + return new pcu::PCU(groupComm); } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} - -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -68,7 +60,7 @@ void getConfig(int argc, char** argv) meshFile = argv[2]; outFile = argv[3]; partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj->Peers()); } } @@ -76,7 +68,8 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -86,19 +79,20 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0); + getConfig(argc,argv,PCUObj.get()); + bool isOriginal = ((PCUObj.get()->Self() % partitionFactor) == 0); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - switchToOriginals(); + pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj.get()); if (isOriginal) { - m = apf::loadMdsMesh(g, meshFile); + m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } - switchToAll(); - m = repeatMdsMesh(m, g, plan, partitionFactor); + if(m != nullptr) m->switchPCU(PCUObj.get()); + delete groupedPCUObj; + m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get(), groupedPCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); @@ -108,7 +102,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ugrid.cc b/test/ugrid.cc index 87ef32016..77adb8327 100644 --- a/test/ugrid.cc +++ b/test/ugrid.cc @@ -19,56 +19,51 @@ apf::Migration* getPlan(apf::Mesh* m, const int partitionFactor) return plan; } -void switchToOriginals(const int partitionFactor) +pcu::PCU* getGroupedPCU(const int partitionFactor, pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + return new pcu::PCU(groupComm); } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); const int partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); - bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj.get()->Peers()); + bool isOriginal = ((PCUObj.get()->Self() % partitionFactor) == 0); gmi_model* g = gmi_load(".null"); apf::Mesh2* m = 0; apf::Migration* plan = 0; - switchToOriginals(partitionFactor); + pcu::PCU *groupedPCUObj = getGroupedPCU(partitionFactor, PCUObj.get()); if (isOriginal) { - m = apf::loadMdsFromUgrid(g, argv[1]); + m = apf::loadMdsFromUgrid(g, argv[1], groupedPCUObj); apf::deriveMdsModel(m); m->verify(); plan = getPlan(m, partitionFactor); } - switchToAll(); - m = repeatMdsMesh(m, g, plan, partitionFactor); + if(m != nullptr) m->switchPCU(PCUObj.get()); + delete groupedPCUObj; + m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); Parma_PrintPtnStats(m, ""); gmi_write_dmg(g,argv[2]); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/verify.cc b/test/verify.cc index 4e2b39ffe..1343d443a 100644 --- a/test/verify.cc +++ b/test/verify.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -11,12 +10,14 @@ #include #endif #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +27,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); m->verify(); m->destroyNative(); apf::destroyMesh(m); @@ -36,7 +37,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/verify_convert.cc b/test/verify_convert.cc index 13f527f23..50136fd38 100644 --- a/test/verify_convert.cc +++ b/test/verify_convert.cc @@ -2,7 +2,6 @@ * this test verifies that the convert process properly clones the underlying * fields, numberings, and tags */ -#include #include #include #include @@ -12,14 +11,16 @@ #include #include #include -apf::Mesh2* createEmptyMesh() +#include + +apf::Mesh2* createEmptyMesh(pcu::PCU *PCUObj) { gmi_model* mdl = gmi_load(".null"); - return apf::makeEmptyMdsMesh(mdl, 1, false); + return apf::makeEmptyMdsMesh(mdl, 1, false, PCUObj); } -apf::Mesh2* createMesh() +apf::Mesh2* createMesh(pcu::PCU *PCUObj) { - apf::Mesh2* m = createEmptyMesh(); + apf::Mesh2* m = createEmptyMesh(PCUObj); apf::Vector3 pts[2] = {apf::Vector3(0,0,0), apf::Vector3(1,0,0)}; apf::MeshEntity* verts[2]; for( int i=0; i<2; i++) @@ -43,13 +44,14 @@ class twox : public apf::Function { int main(int argc, char* argv[]) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_null(); // create meshes and write data to one of them - apf::Mesh* m1 = createMesh(); - apf::Mesh2* m2 = createEmptyMesh(); + apf::Mesh* m1 = createMesh(PCUObj.get()); + apf::Mesh2* m2 = createEmptyMesh(PCUObj.get()); // create field on m1 apf::Field* f = apf::createLagrangeField(m1, "field1", apf::SCALAR, 1); apf::Function* func = new twox(f); @@ -137,7 +139,7 @@ int main(int argc, char* argv[]) m2->end(it); // check that not transfering Fields/Numberings/Tags also works - apf::Mesh2* m3 = createEmptyMesh(); + apf::Mesh2* m3 = createEmptyMesh(PCUObj.get()); apf::convert(m1, m3, NULL, NULL, false); m3->verify(); @@ -160,7 +162,7 @@ int main(int argc, char* argv[]) apf::destroyMesh(m1); apf::destroyMesh(m2); apf::destroyMesh(m3); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/visualizeAnisoSizes.cc b/test/visualizeAnisoSizes.cc index cb3516e66..b145aee2d 100644 --- a/test/visualizeAnisoSizes.cc +++ b/test/visualizeAnisoSizes.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX @@ -18,6 +17,7 @@ #include #include #include +#include // === includes for safe_mkdir === #include @@ -37,16 +37,18 @@ void visualizeSizeField( const char* frameName, int sampleSize[2], double userScale, - const char* outputPrefix); + const char* outputPrefix, + pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if (argc < 8) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE1: %s " " \n", argv[0]); printf("USAGE2: %s " @@ -77,7 +79,7 @@ int main(int argc, char** argv) int sampleSize[2] = {atoi(argv[5]),atoi(argv[6])}; double scale = atof(argv[7]); safe_mkdir(inPrefix); - visualizeSizeField(".null", meshFile, sizeName, frameName, sampleSize, scale, inPrefix); + visualizeSizeField(".null", meshFile, sizeName, frameName, sampleSize, scale, inPrefix, PCUObj.get()); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -86,7 +88,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -131,19 +133,20 @@ void visualizeSizeField( const char* frameName, int sampleSize[2], double userScale, - const char* outputPrefix) + const char* outputPrefix, + pcu::PCU *PCUObj) { apf::Mesh2* m; #ifdef HAVE_SIMMETRIX /* if it is a simmetrix mesh */ if (ph::mesh_has_ext(meshFile, "sms")) { pParMesh sim_mesh = PM_load(meshFile, NULL, NULL); - m = apf::createMesh(sim_mesh); + m = apf::createMesh(sim_mesh, PCUObj); } else #endif { // load the mesh change to desired order and write as before vtks - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); } m->verify(); diff --git a/test/viz.cc b/test/viz.cc index 2bc6e9061..28a6b5349 100644 --- a/test/viz.cc +++ b/test/viz.cc @@ -3,9 +3,9 @@ #include #include #include -#include #include #include +#include #include "../viz/viz.h" namespace { @@ -37,11 +37,12 @@ int main(int argc, char** argv) int provided; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided); PCU_ALWAYS_ASSERT(provided==MPI_THREAD_MULTIPLE); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); Visualization v; @@ -105,6 +106,6 @@ int main(int argc, char** argv) v.breakpoint("Part Boundaries"); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxBalance.cc b/test/vtxBalance.cc index cca54d464..6029c2cf9 100644 --- a/test/vtxBalance.cc +++ b/test/vtxBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #endif #include #include +#include namespace { apf::MeshTag* setVtxWeights(apf::Mesh* m) { @@ -31,10 +31,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -48,7 +49,7 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setVtxWeights(m); const double step = 0.5; const int verbose = 1; @@ -68,6 +69,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxEdgeElmBalance.cc b/test/vtxEdgeElmBalance.cc index 39cce7e03..7cce097b1 100644 --- a/test/vtxEdgeElmBalance.cc +++ b/test/vtxEdgeElmBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -13,6 +12,7 @@ #endif #include #include +#include namespace { void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim, double w=1.0) { @@ -46,10 +46,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 6); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 6 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -64,7 +65,7 @@ int main(int argc, char** argv) gmi_register_mesh(); //load model and mesh double targetImb = atof(argv[5]); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); apf::MeshTag* weights = setWeights(m,atof(argv[4])); Parma_PrintWeightedPtnStats(m, weights, "initial"); const double step = 0.5; const int verbose = 1; @@ -86,6 +87,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxElmBalance.cc b/test/vtxElmBalance.cc index 20d3eff8a..88f52ce29 100644 --- a/test/vtxElmBalance.cc +++ b/test/vtxElmBalance.cc @@ -3,10 +3,10 @@ #include #include #include -#include #include #include #include +#include namespace { void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) { @@ -35,17 +35,18 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setWeights(m); const double step = 0.5; const int verbose = 2; @@ -59,6 +60,6 @@ int main(int argc, char** argv) // destroy mds m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxElmMixedBalance.cc b/test/vtxElmMixedBalance.cc index 54e1b7169..a033c91cd 100644 --- a/test/vtxElmMixedBalance.cc +++ b/test/vtxElmMixedBalance.cc @@ -2,9 +2,9 @@ #include #include #include -#include #include #include +#include #ifdef HAVE_SIMMETRIX #include #include @@ -18,7 +18,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -28,7 +29,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); m->verify(); ma::localizeLayerStacks(m); m->verify(); @@ -41,7 +42,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/writeIPFieldTest.cc b/test/writeIPFieldTest.cc index 7a45027d7..37a62a12b 100644 --- a/test/writeIPFieldTest.cc +++ b/test/writeIPFieldTest.cc @@ -2,23 +2,24 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); m->verify(); apf::createIPField(m, "Cauchy_Stress", apf::MATRIX, 1); apf::writeVtkFiles("mesh_with_IPField", m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/writePart.cc b/test/writePart.cc index 0f44254f2..2aaa9f78b 100644 --- a/test/writePart.cc +++ b/test/writePart.cc @@ -2,28 +2,29 @@ #include #include #include -#include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); if (m->getPCU()->Self() == atoi(argv[3])) apf::writeMdsPart(m, argv[4]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/writeVtxPtn.cc b/test/writeVtxPtn.cc index b9835cdb7..58adae5f2 100644 --- a/test/writeVtxPtn.cc +++ b/test/writeVtxPtn.cc @@ -2,11 +2,11 @@ #include #include #include -#include #include #include #include #include +#include namespace { const char* modelFile = 0; @@ -18,10 +18,10 @@ namespace { apf::destroyMesh(m); } - void getConfig(int argc, char** argv) + void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -34,13 +34,14 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/xgc_split.cc b/test/xgc_split.cc index 31acf3819..e69cb2e44 100644 --- a/test/xgc_split.cc +++ b/test/xgc_split.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include "apfMDS.h" #include "apfShape.h" @@ -55,22 +56,23 @@ Migration* get_xgc_plan(pGeom g, pMesh m) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - getConfig(argc,argv,PCUObj); + getConfig(argc,argv,PCUObj.get()); - pGeom g = pumi_geom_load(modelFile, PCUObj); + pGeom g = pumi_geom_load(modelFile, PCUObj.get()); pMesh m; if (serial) { - m = pumi_mesh_loadSerial(g, meshFile, PCUObj); + m = pumi_mesh_loadSerial(g, meshFile, PCUObj.get()); // split a serial mesh based on model ID Migration* plan = get_xgc_plan(g, m); pumi_mesh_migrate(m, plan); pumi_mesh_write(m, outFile); } else - m = pumi_mesh_load(g, meshFile, pumi_size(PCUObj), PCUObj); + m = pumi_mesh_load(g, meshFile, pumi_size(PCUObj.get()), PCUObj.get()); // write to vtk char without_extension[256]; @@ -93,7 +95,7 @@ int main(int argc, char** argv) pumi_mesh_delete(m); - pumi_finalize(PCUObj); + } MPI_Finalize(); } diff --git a/test/zbalance.cc b/test/zbalance.cc index b0fac0263..85a49a934 100644 --- a/test/zbalance.cc +++ b/test/zbalance.cc @@ -5,10 +5,10 @@ #include #include #include -#include #include #include #include // exit and EXIT_FAILURE +#include #ifdef HAVE_SIMMETRIX #include #include @@ -20,10 +20,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.get()->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -38,7 +39,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); apf::MeshTag* weights = Parma_WeighByMemory(m); apf::Balancer* balancer = makeZoltanBalancer(m, apf::GRAPH, apf::REPARTITION); balancer->balance(weights, 1.10); @@ -56,6 +57,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/zsplit.cc b/test/zsplit.cc index 50b8259fb..1a7d02867 100644 --- a/test/zsplit.cc +++ b/test/zsplit.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -14,6 +13,7 @@ #include #include #include +#include namespace { @@ -40,28 +40,20 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -void switchToOriginals() +pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + return new pcu::PCU(groupComm); } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} - -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -70,7 +62,7 @@ void getConfig(int argc, char** argv) meshFile = argv[2]; outFile = argv[3]; partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj->Peers()); } } @@ -78,7 +70,8 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -88,19 +81,20 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0); + getConfig(argc,argv,PCUObj.get()); + bool isOriginal = ((PCUObj.get()->Self() % partitionFactor) == 0); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - switchToOriginals(); + pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj.get()); if (isOriginal) { - m = apf::loadMdsMesh(g, meshFile); + m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } - switchToAll(); - m = apf::repeatMdsMesh(m, g, plan, partitionFactor); + if(m != nullptr) m->switchPCU(PCUObj.get()); + delete groupedPCUObj; + m = apf::repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); @@ -110,6 +104,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } From 7dbae41eb5cf4131cca898d1a151e565bf7ce532 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 29 Apr 2024 09:28:47 -0400 Subject: [PATCH 109/141] Added comments for uses of switchPCU --- test/fusion.cc | 2 ++ test/ptnParma.cc | 4 +++- test/split.cc | 2 ++ test/ugrid.cc | 2 ++ test/zsplit.cc | 2 ++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/fusion.cc b/test/fusion.cc index 7c341fea3..ba91827da 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -164,6 +164,8 @@ int main( int argc, char* argv[]) code.meshFile = argv[1]; apf::Unmodulo outMap(pcu_obj.get()->Self(), 2); Parma_SplitPartition(nullptr, 2, code, pcu_obj.get()); + //Have to call switchPCU here because the mesh needed to be made inside GroupCode run() + //and inside parma_group.cc runInGroups() we set code.PCUObj to groupedPCU code.mesh->switchPCU(pcu_obj.get()); apf::remapPartition(code.mesh, outMap); code.mesh->verify(); diff --git a/test/ptnParma.cc b/test/ptnParma.cc index 701143907..71eff01f3 100644 --- a/test/ptnParma.cc +++ b/test/ptnParma.cc @@ -127,8 +127,10 @@ void mymain(bool ismaster, pcu::PCU *PCUObj) Parma_PrintPtnStats(m, "initial"); plan = getPlan(m); } - delete groupedPCUObj; + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU if(m != nullptr) m->switchPCU(PCUObj); + delete groupedPCUObj; m = apf::repeatMdsMesh(m, g, plan, partitionFactor, PCUObj); runParma(m); m->writeNative(outFile); diff --git a/test/split.cc b/test/split.cc index 8855485dc..fe1f6925a 100644 --- a/test/split.cc +++ b/test/split.cc @@ -90,6 +90,8 @@ int main(int argc, char** argv) m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU if(m != nullptr) m->switchPCU(PCUObj.get()); delete groupedPCUObj; m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get(), groupedPCUObj); diff --git a/test/ugrid.cc b/test/ugrid.cc index 77adb8327..17bf7b991 100644 --- a/test/ugrid.cc +++ b/test/ugrid.cc @@ -55,6 +55,8 @@ int main(int argc, char** argv) m->verify(); plan = getPlan(m, partitionFactor); } + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU if(m != nullptr) m->switchPCU(PCUObj.get()); delete groupedPCUObj; m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); diff --git a/test/zsplit.cc b/test/zsplit.cc index 1a7d02867..8011fba86 100644 --- a/test/zsplit.cc +++ b/test/zsplit.cc @@ -92,6 +92,8 @@ int main(int argc, char** argv) m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU if(m != nullptr) m->switchPCU(PCUObj.get()); delete groupedPCUObj; m = apf::repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); From 2b03519864e943f9ef8aceed4b243276e31de9f5 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 29 Apr 2024 09:40:46 -0400 Subject: [PATCH 110/141] added back reeResidualFunctionals.cc --- parma/group/parma_group.cc | 2 - ree/reeResidualFunctionals.cc | 924 ++++++++++++++++++++++++++++++++++ 2 files changed, 924 insertions(+), 2 deletions(-) diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index 3cbbdfe68..ba43cb1c5 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -137,7 +137,5 @@ void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu apf::Unmodulo outMap(PCUObj->Self(), factor); runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); } - - //runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); } diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc index e69de29bb..3016071b5 100644 --- a/ree/reeResidualFunctionals.cc +++ b/ree/reeResidualFunctionals.cc @@ -0,0 +1,924 @@ +/* + * Copyright (C) 2011 Scientific Computation Research Center + * + * This work is open source software, licensed under the terms of the + * BSD license as described in the LICENSE file in the top-level directory. + */ +#include +#include +#include "apfElement.h" +#include "ree.h" + +namespace ree { + +enum {VISITED}; + +/* overall information useful during equilibration */ +struct Equilibration { + apf::Mesh* mesh; + /* mesh dimension, so far handling 3 only */ + int dim; + /* polynomial order of Nedelec space */ + int order; + /* input scalar field containing Nedelec dofs for solution electric field */ + apf::Field* ef; + /* tags each edge once it has been visited during the equilibration process */ + apf::MeshTag* tag; + /* output field containing correction values. + * currently 3 scalar values are stored on each face + * in order corresponding to downward edges of the face */ + apf::Field* g; +}; + +static void setupEquilibration(Equilibration* eq, apf::Field* f, apf::Field* g) +{ + eq->mesh = apf::getMesh(f); + eq->tag = eq->mesh->createIntTag("isVisited", 1); + eq->dim = eq->mesh->getDimension(); + eq->ef = f; + eq->order = f->getShape()->getOrder(); + eq->g = g; + + double zeros[3] = {0., 0., 0.}; + apf::MeshEntity* face; + apf::MeshIterator* it = eq->mesh->begin(2); + while ((face = eq->mesh->iterate(it))) { + apf::setComponents(eq->g, face, 0, zeros); + } + eq->mesh->end(it); +} + +struct QRDecomp { + mth::Matrix Q; + mth::Matrix R; +}; + +typedef std::vector EntityVector; + +struct EdgePatch { + apf::Mesh* mesh; + Equilibration* equilibration; + /* the edge entity around which the patch + is centered. a patch collects entities + (faces & tets) around this edge entity */ + apf::MeshEntity* entity; + bool isOnBdry; + EntityVector tets; + EntityVector faces; + mth::Matrix A; + mth::Matrix At; + mth::Matrix T; // T = A*At + 1 + mth::Vector b; + mth::Vector x; + QRDecomp qr; +}; + +static void setupEdgePatch(EdgePatch* ep, Equilibration* eq) +{ + ep->mesh = eq->mesh; + ep->equilibration = eq; + ep->entity = 0; +} + +static void startEdgePatch(EdgePatch* ep, apf::MeshEntity* e) +{ + ep->tets.clear(); + ep->faces.clear(); + ep->entity = e; + ep->isOnBdry = isOnDomainBoundary(ep->mesh, ep->entity); +} + +static void addEntityToPatch(EdgePatch* ep, apf::MeshEntity* e) +{ + if(ep->mesh->getType(e) == apf::Mesh::TRIANGLE) + ep->faces.push_back(e); + if(ep->mesh->getType(e) == apf::Mesh::TET) + ep->tets.push_back(e); +} + +static void addEntitiesToPatch( + EdgePatch* ep, apf::DynamicArray& es) +{ + for (std::size_t i=0; i < es.getSize(); ++i) + addEntityToPatch(ep, es[i]); +} + +static bool getInitialEdgePatch(EdgePatch* ep, apf::CavityOp* o) +{ + if ( ! o->requestLocality(&ep->entity,1)) + return false; + apf::DynamicArray adjacent; + ep->mesh->getAdjacent(ep->entity, 3, adjacent); + addEntitiesToPatch(ep, adjacent); + + ep->mesh->getAdjacent(ep->entity, 2, adjacent); + addEntitiesToPatch(ep, adjacent); + + return true; +} + +static bool buildEdgePatch(EdgePatch* ep, apf::CavityOp* o) +{ + if (!getInitialEdgePatch(ep, o)) return false; + return true; +} + +/* + * Reference: Leszek Demkowicz, Computing with hp-adaptive + * finite elements, vol2, equation (11.118, 11.119, 11.122). + */ +static void assembleEdgePatchLHS(EdgePatch* ep) +{ + int ne = ep->tets.size(); + int nf = ep->faces.size(); + if( isOnDomainBoundary(ep->mesh, ep->entity) ) { + ep->T.resize(ne+nf, ne+nf); + ep->T.zero(); + for (int i = 0; i < nf; i++) + ep->T(i,i) = 2.; + for (int i = 0; i < ne-1; i++) { + ep->T(i+nf,i) = 1.; ep->T(i+nf,i+1) = -1.; + ep->T(i,i+nf) = 1.; ep->T(i+1,i+nf) = -1.; + } + ep->T(ne+nf-1, ne-1) = 1.; ep->T(ne+nf-1, ne) = 1.; + ep->T(ne-1, ne+nf-1) = 1.; ep->T(ne, ne+nf-1) = 1.; + } + else if( ! isOnDomainBoundary(ep->mesh, ep->entity) ) { + ep->A.resize(ne, nf); + ep->A.zero(); + for (int i = 0; i < ne-1; i++) { + ep->A(i,i) = 1.; ep->A(i,i+1) = -1.; + } + ep->A(ne-1,0) = -1.; ep->A(ne-1,ne-1) = 1.; + // A is singular so do (A*At) + 1.0 + // to pick a particular solution + ep->At.resize(nf,ne); + mth::transpose(ep->A, ep->At); + mth::multiply(ep->A, ep->At, ep->T); + + for (int i = 0; i < ne; i++) + for (int j = 0; j < ne; j++) + ep->T(i,j) += 1.; + } + mth::decomposeQR(ep->T, ep->qr.Q, ep->qr.R); +} + +/* + * Performs Curl Curl integration using curl vector Nedelec shapes + */ +void assembleCurlCurlElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e, + apf::Field* f, mth::Matrix& elmat) +{ + apf::FieldShape* fs = f->getShape(); + int type = mesh->getType(e); + PCU_ALWAYS_ASSERT(type == apf::Mesh::TET); + int nd = apf::countElementNodes(fs, type); + int dim = apf::getDimension(mesh, e); + int dimc = (dim == 3) ? 3 : 1; + double w; + + apf::NewArray curlshape(nd); + mth::Matrix phys_curlshape(nd, dimc); + elmat.resize(nd,nd); + + apf::MeshElement* me = apf::createMeshElement(mesh, e); + apf::Element* el = apf::createElement(f, me); + int int_order = 2 * fs->getOrder() - 2; + int np = apf::countIntPoints(me, int_order); + + elmat.zero(); + apf::Vector3 p; + for (int i = 0; i < np; i++) { + apf::getIntPoint(me, int_order, i, p); + double weight = apf::getIntWeight(me, int_order, i); + apf::Matrix3x3 J; + apf::getJacobian(me, p, J); + double jdet = apf::getJacobianDeterminant(J, dim); + w = weight / jdet; + + if (dim == 3) { + el->getShape()->getLocalVectorCurls(mesh, e, p, curlshape); + phys_curlshape.zero(); + for (int j = 0; j < nd; j++) + for (int k = 0; k < dim; k++) + for (int l = 0; l < dim; l++) + phys_curlshape(j,k) += curlshape[j][l] * J[l][k]; + } + mth::Matrix phys_curlshapeT; + mth::transpose(phys_curlshape, phys_curlshapeT); + + mth::Matrix M (nd, nd); + M.zero(); + mth::multiply(phys_curlshape, phys_curlshapeT, M); + M *= w; + elmat += M; + } + apf::destroyElement(el); + apf::destroyMeshElement(me); +} + +/* + * Performs Vector Vector Mass integration using vector Nedelec shapes + */ +void assembleVectorMassElementMatrix(apf::Mesh* mesh, apf::MeshEntity* e, + apf::Field* f, mth::Matrix& elmat) +{ + apf::FieldShape* fs = f->getShape(); + int type = mesh->getType(e); + PCU_ALWAYS_ASSERT(type == apf::Mesh::TET || type == apf::Mesh::TRIANGLE); + int nd = apf::countElementNodes(fs, type); + int dim = apf::getDimension(mesh, e); + int sdim = mesh->getDimension(); + double w; + + apf::NewArray vectorshapes(nd); + elmat.resize(nd,nd); + + apf::MeshElement* me = apf::createMeshElement(mesh, e); + apf::Element* el = apf::createElement(f, me); + int int_order = 2 * fs->getOrder(); + int np = apf::countIntPoints(me, int_order); + + elmat.zero(); + apf::Vector3 p; + for (int i = 0; i < np; i++) { + apf::getIntPoint(me, int_order, i, p); + double weight = apf::getIntWeight(me, int_order, i); + apf::Matrix3x3 J; + apf::getJacobian(me, p, J); + double jdet = apf::getJacobianDeterminant(J, dim); + w = weight * jdet; + + apf::getVectorShapeValues(el, p, vectorshapes); + mth::Matrix vectorShapes (nd, sdim); + for (int j = 0; j < nd; j++) + for (int k = 0; k < sdim; k++) + vectorShapes(j,k) = vectorshapes[j][k]; + + mth::Matrix vectorShapesT (sdim, nd); + mth::transpose(vectorShapes, vectorShapesT); + + mth::Matrix M (nd,nd); + M.zero(); + mth::multiply(vectorShapes, vectorShapesT, M); + M *= w; + elmat += M; + } + + apf::destroyElement(el); + apf::destroyMeshElement(me); +} + +void assembleElementMatrix(apf::Mesh* mesh, apf::MeshEntity*e, + apf::Field* f, mth::Matrix& elmat) +{ + mth::Matrix curl_elmat, mass_elmat; + assembleCurlCurlElementMatrix(mesh, e, f, curl_elmat); + assembleVectorMassElementMatrix(mesh, e, f, mass_elmat); + + elmat.resize(curl_elmat.rows(), curl_elmat.cols()); + elmat.zero(); + elmat += curl_elmat; + elmat += mass_elmat; +} + +/* + * computes local bilinear form integral restricted to + * an edge of a tet element + */ +static double getLocalEdgeBLF(EdgePatch* ep, apf::MeshEntity* tet) +{ + PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET); + // findIn edge in downward edges of tet + apf::Downward e; + int ne = ep->mesh->getDownward(tet, 1, e); + int ei = apf::findIn(e, ne, ep->entity); + // get Element Dofs + apf::MeshElement* me = apf::createMeshElement(ep->mesh, tet); + apf::Element* el = apf::createElement(ep->equilibration->ef, me); + int type = ep->mesh->getType(tet); + int nd = apf::countElementNodes(el->getFieldShape(), type); + apf::NewArray d (nd); + el->getElementNodeData(d); + mth::Vector dofs (nd); + for (int i = 0; i < nd; i++) + dofs(i) = d[i]; + // assemble curl curl element matrix + mth::Matrix curl_elmat; + assembleCurlCurlElementMatrix(ep->mesh, tet, + ep->equilibration->ef, curl_elmat); + // assemble vector mass element matrix + mth::Matrix mass_elmat; + assembleVectorMassElementMatrix(ep->mesh, tet, + ep->equilibration->ef, mass_elmat); + // add element matrices + mth::Matrix elmat(nd, nd); + elmat.zero(); + elmat += curl_elmat; + elmat += mass_elmat; + // multiply element matrix with element dofs + mth::Vector blf_integrals (nd); + mth::multiply(elmat, dofs, blf_integrals); + + apf::destroyElement(el); + apf::destroyMeshElement(me); + + // pick edge index from the resulting vector + // negation of negative ND dofs + int which, rotate; bool flip; + apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate); + if (flip) + blf_integrals(ei) = -1*blf_integrals(ei); + return blf_integrals(ei); +} + +/* + * Capability can be added in the future to allow the user + * to define a custom function */ +void pumiUserFunction(apf::Mesh* mesh, apf::MeshEntity* e, + const apf::Vector3& x, mth::Vector& f) +{ + double freq = 1.; + double kappa = freq * M_PI; + int dim = apf::getDimension(mesh, e); + if (dim == 3) { + f(0) = (1. + kappa * kappa) * sin(kappa * x[1]); + f(1) = (1. + kappa * kappa) * sin(kappa * x[2]); + f(2) = (1. + kappa * kappa) * sin(kappa * x[0]); + /*f(0) = 0.; + f(1) = 0.; + f(2) = 0.;*/ + } + else { + f(0) = (1. + kappa * kappa) * sin(kappa * x[1]); + f(1) = (1. + kappa * kappa) * sin(kappa * x[0]); + f(2) = 0.0; + } +} +void assembleDomainLFElementVector(apf::Mesh* mesh, apf::MeshEntity* e, + apf::Field* f, mth::Vector& elvect) +{ + apf::FieldShape* fs = f->getShape(); + int type = mesh->getType(e); + PCU_ALWAYS_ASSERT(type == apf::Mesh::TET); + int nd = apf::countElementNodes(fs, type); + int dim = apf::getDimension(mesh, e); + double w; + + apf::NewArray vectorshapes(nd); + elvect.resize(nd); + mth::Vector val (dim); + val.zero(); + apf::Vector3 p; + + apf::MeshElement* me = apf::createMeshElement(mesh, e); + apf::Element* el = apf::createElement(f, me); + int int_order = 2 * fs->getOrder(); + int np = apf::countIntPoints(me, int_order); + + elvect.zero(); + for (int i = 0; i < np; i++) { + apf::getIntPoint(me, int_order, i, p); + double weight = apf::getIntWeight(me, int_order, i); + apf::Matrix3x3 J; + apf::getJacobian(me, p, J); + double jdet = apf::getJacobianDeterminant(J, dim); + w = weight * jdet; + + apf::getVectorShapeValues(el, p, vectorshapes); + apf::Vector3 global; + apf::mapLocalToGlobal(me, p, global); + pumiUserFunction(mesh, e, global, val); + val *= w; + + mth::Matrix vectorShapes (nd, dim); + for (int j = 0; j < nd; j++) + for (int k = 0; k < dim; k++) + vectorShapes(j,k) = vectorshapes[j][k]; + mth::Vector V (nd); + V.zero(); + mth::multiply(vectorShapes, val, V); + elvect += V; + } + + apf::destroyElement(el); + apf::destroyMeshElement(me); +} + +/* + * computes local linear form integral restricted to + * an edge of a tet element + */ +static double getLocalEdgeLF(EdgePatch* ep, apf::MeshEntity* tet) +{ + PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET); + // findIn edge in downward edges of tet + apf::Downward e; + int ne = ep->mesh->getDownward(tet, 1, e); + int ei = apf::findIn(e, ne, ep->entity); + // assemble Domain LF Vector + mth::Vector elvect; + assembleDomainLFElementVector(ep->mesh, tet, + ep->equilibration->ef, elvect); + int which, rotate; bool flip; + apf::getAlignment(ep->mesh, tet, ep->entity, which, flip, rotate); + if (flip) + elvect(ei) = -1*elvect(ei); + return elvect(ei); +} + +/* + * Given a tet and one of its faces, the vertex of the tet + * opposite to the given face is returned. + */ +static apf::MeshEntity* getTetOppVert( + apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f) +{ + apf::Downward fvs; + int fnv = m->getDownward(f, 0, fvs); + apf::Downward tvs; + int tnv = m->getDownward(t, 0, tvs); + PCU_ALWAYS_ASSERT(tnv == 4 && fnv == 3); + for (int i = 0; i < tnv; i++) { + if (apf::findIn(fvs, fnv, tvs[i]) == -1) + return tvs[i]; + } + return 0; +} + +/* + * Given a face and one of its edges, the vertex of the face + * opposite to the given edge is returned. + */ +static apf::MeshEntity* getFaceOppVert( + apf::Mesh* m, apf::MeshEntity* f, apf::MeshEntity* e) +{ + apf::Downward evs; + int env = m->getDownward(e, 0, evs); + apf::Downward fvs; + int fnv = m->getDownward(f, 0, fvs); + PCU_ALWAYS_ASSERT(env == 2 && fnv == 3); + for (int i = 0; i < fnv; i++) { + if (apf::findIn(evs, env, fvs[i]) == -1) + return fvs[i]; + } + return 0; +} + +static apf::Vector3 computeFaceNormal( + apf::Mesh* m, apf::MeshEntity* f, apf::Vector3 const& p) +{ + // Compute face normal using face Jacobian, + // so it can also be used for curved mesh elements + apf::MeshElement* me = apf::createMeshElement(m, f); + apf::Matrix3x3 J; + apf::getJacobian(me, p, J); + apf::destroyMeshElement(me); + + apf::Vector3 g1 = J[0]; + apf::Vector3 g2 = J[1]; + apf::Vector3 n = apf::cross( g1, g2 ); + return n.normalize(); +} + +apf::Vector3 computeFaceOutwardNormal(apf::Mesh* m, + apf::MeshEntity* t, apf::MeshEntity* f, apf::Vector3 const& p) +{ + apf::Vector3 n = computeFaceNormal(m, f, p); + + // orient the normal outwards from the tet + apf::MeshEntity* oppVert = getTetOppVert(m, t, f); + apf::Vector3 vxi = apf::Vector3(0.,0.,0.); + + apf::Vector3 txi; + m->getPoint(oppVert, 0, txi); + + apf::MeshElement* fme = apf::createMeshElement(m, f); + apf::Vector3 pxi; + apf::mapLocalToGlobal(fme, p, pxi); + apf::destroyMeshElement(fme); + + apf::Vector3 pxiTotxi = txi - pxi; + if (pxiTotxi*n > 0) { + n = n*-1.; + } + return n; +} + +// TODO this needs generalisation for cases with internal model boundaries +// (i.e., interfaces) +bool isOnDomainBoundary(apf::Mesh* m, apf::MeshEntity* e) +{ + return m->getModelType(m->toModel(e)) < m->getDimension(); +} + +/* + * computes local flux integral restricted to + * an edge of a tet element + */ +static double getLocalFluxIntegral(EdgePatch* ep, apf::MeshEntity* tet) +{ + PCU_ALWAYS_ASSERT(ep->mesh->getType(tet) == apf::Mesh::TET); + + double fluxIntegral = 0.0; + // 1. get faces of the tet in the patch + apf::Downward f; + int nf = ep->mesh->getDownward(tet, 2, f); + std::vector patchFaces; + for (int i = 0; i < nf; i++) { + if(std::find(ep->faces.begin(), ep->faces.end(), f[i]) != ep->faces.end()) + patchFaces.push_back(f[i]); + } + PCU_ALWAYS_ASSERT(patchFaces.size() == 2); + + // 2. loop over the patch faces + for (unsigned int i = 0; i < patchFaces.size(); i++) { + double fluxFaceIntegral = 0.0; + // 3. get upward tets of the current face + apf::Up up; + apf::MeshEntity* currentFace = patchFaces[i]; + ep->mesh->getUp(currentFace, up); + if (isOnDomainBoundary(ep->mesh, currentFace)) + PCU_ALWAYS_ASSERT( up.n == 1); + else + PCU_ALWAYS_ASSERT( up.n == 2); + + apf::MeshEntity* firstTet = up.e[0]; + apf::MeshEntity* secondTet = nullptr; + if (up.n == 2) + secondTet = up.e[1]; + + // 4. findIn edge in downward edges of current face + apf::Downward e; + int ne = ep->mesh->getDownward(currentFace, 1, e); + PCU_ALWAYS_ASSERT(ne == 3); + int ei = apf::findIn(e, ne, ep->entity); + + // 5. count integration points for flux face integral + apf::FieldShape* fs = ep->equilibration->ef->getShape(); + int int_order = 2 * fs->getOrder(); + apf::MeshElement* fme = apf::createMeshElement(ep->mesh, currentFace); + apf::Element* fel = apf::createElement(ep->equilibration->ef, fme); + int np = apf::countIntPoints(fme, int_order); + + // loop over integration points + apf::Vector3 p, tet1xi, tet2xi, curl1, curl2, curl, + fn1, fn2, tk, vshape; + for (int n = 0; n < np; n++) { + apf::getIntPoint(fme, int_order, n, p); + double weight = apf::getIntWeight(fme, int_order, n); + apf::Matrix3x3 fJ; + apf::getJacobian(fme, p, fJ); + double jdet = apf::getJacobianDeterminant( + fJ, apf::getDimension(ep->mesh, currentFace)); + + // compute face outward normals wrt tets + if (tet == firstTet) { + fn1 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p); + fn2 = apf::Vector3(0.,0.,0.); + } + else { + fn1 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p); + fn2 = apf::Vector3(0.,0.,0.); + } + if (up.n == 2) { + if (tet == firstTet) + fn2 = computeFaceOutwardNormal(ep->mesh, secondTet, currentFace, p); + else + fn2 = computeFaceOutwardNormal(ep->mesh, firstTet, currentFace, p); + } + + curl.zero(); + // compute curl1 + tet1xi = apf::boundaryToElementXi(ep->mesh, currentFace, firstTet, p); + apf::MeshElement* me1 = apf::createMeshElement(ep->mesh, firstTet); + apf::Element* el1 = apf::createElement(ep->equilibration->ef, me1); + apf::getCurl(el1, tet1xi, curl1); + apf::Vector3 temp1 = apf::cross(fn1, curl1); + curl += temp1; + apf::destroyElement(el1); + apf::destroyMeshElement(me1); + + // compute curl2 + if (up.n == 2) { + tet2xi = apf::boundaryToElementXi(ep->mesh, currentFace, secondTet, p); + apf::MeshElement* me2 = apf::createMeshElement(ep->mesh, secondTet); + apf::Element* el2 = apf::createElement(ep->equilibration->ef, me2); + apf::getCurl(el2, tet2xi, curl2); + apf::Vector3 temp2 = apf::cross(fn2, curl2); + curl += (temp2 * -1.); + curl = curl * 1./2.; + apf::destroyElement(el2); + apf::destroyMeshElement(me2); + } + + // compute tk (inter-element averaged flux) + tk = curl; + + // compute vector shape + int type = apf::Mesh::TRIANGLE; + int nd = apf::countElementNodes(fs, type); + apf::NewArray vectorshapes (nd); + apf::getVectorShapeValues(fel, p, vectorshapes); + vshape = vectorshapes[ei]; + + // compute integral + fluxFaceIntegral += (tk * vshape) * weight * jdet; + } + apf::destroyElement(fel); + apf::destroyMeshElement(fme); + + fluxIntegral += fluxFaceIntegral; + } + return fluxIntegral; +} + +static void assembleEdgePatchRHS(EdgePatch* p) +{ + if (p->isOnBdry) { + p->b.resize(p->tets.size() + p->faces.size()); + p->b.zero(); + } + else { + p->b.resize(p->tets.size()); + p->b.zero(); + } + int ne = p->tets.size(); + int nf = p->faces.size(); + for (int i = 0; i < ne; i++) { + apf::MeshEntity* tet = p->tets[i]; + double blfIntegral = getLocalEdgeBLF(p, tet); + double lfIntegral = getLocalEdgeLF(p, tet); + double fluxIntegral = getLocalFluxIntegral(p, tet); + if(p->isOnBdry) + p->b(nf+i) = blfIntegral - lfIntegral - fluxIntegral; + else + p->b(i) = blfIntegral - lfIntegral - fluxIntegral; + } +} + +static apf::MeshEntity* getTetOppFaceSharingEdge( + apf::Mesh* m, apf::MeshEntity* t, apf::MeshEntity* f, apf::MeshEntity* e) +{ + apf::MeshEntity* fs[4]; + m->getDownward(t, 2, fs); + for (int i = 0; i < 4; i++) { + if (fs[i] == f) continue; + apf::MeshEntity* es[3]; + m->getDownward(fs[i], 1, es); + if (apf::findIn(es, 3, e) > -1) + return fs[i]; + } + return 0; +} + +/* + * Orders tets and faces in an edge cavity in a + * clockwise (or ccw) direction around the edge. + */ +static void getOrderedTetsandFaces(apf::Mesh* mesh, apf::MeshEntity* edge, + EntityVector& tets, EntityVector& faces) +{ + tets.clear(); + faces.clear(); + if( ! isOnDomainBoundary(mesh, edge) ) { + apf::MeshEntity* currentFace = mesh->getUpward(edge, 0); + apf::Up up; + mesh->getUp(currentFace, up); + PCU_ALWAYS_ASSERT(up.n == 2); + apf::MeshEntity* firstTet = up.e[0]; + apf::MeshEntity* nextTet = up.e[1]; + tets.push_back(firstTet); + apf::MeshEntity* firstFace = getTetOppFaceSharingEdge(mesh, firstTet, + currentFace, edge); + faces.push_back(firstFace); + + while (nextTet != firstTet) { + tets.push_back(nextTet); + faces.push_back(currentFace); + currentFace = getTetOppFaceSharingEdge(mesh, nextTet, currentFace, edge); + PCU_ALWAYS_ASSERT(currentFace); + apf::Up up; + mesh->getUp(currentFace, up); + PCU_ALWAYS_ASSERT(up.n == 2); + if (nextTet != up.e[0]) + nextTet = up.e[0]; + else + nextTet = up.e[1]; + } + } + else { + apf::Up up; + mesh->getUp(edge, up); + apf::MeshEntity* firstFace = nullptr; + for (int i = 0; i < up.n; i++) { + if ( isOnDomainBoundary(mesh, up.e[i]) ) { + firstFace = up.e[i]; break; + } + } + PCU_ALWAYS_ASSERT(firstFace); + faces.push_back(firstFace); + mesh->getUp(firstFace, up); + PCU_ALWAYS_ASSERT(up.n == 1); + apf::MeshEntity* firstTet = up.e[0]; + tets.push_back(firstTet); + + apf::MeshEntity* nextFace = getTetOppFaceSharingEdge(mesh, firstTet, + firstFace, edge); + apf::MeshEntity* nextTet = firstTet; + mesh->getUp(nextFace, up); + while( up.n == 2) { + faces.push_back(nextFace); + if (nextTet != up.e[0]) + nextTet = up.e[0]; + else + nextTet = up.e[1]; + tets.push_back(nextTet); + + nextFace = getTetOppFaceSharingEdge(mesh, nextTet, nextFace, edge); + mesh->getUp(nextFace, up); + } + faces.push_back(nextFace); + } +} + +void getClockwiseTetsandFaces(EdgePatch* p) +{ + if (p->isOnBdry) { + if (p->tets.size() > 1) { + apf::MeshEntity* secondFace = p->faces[1]; + apf::MeshEntity* firstTet = p->tets[0]; + apf::MeshEntity* secondTet = p->tets[1]; + + apf::Downward firstTetFaces, secondTetFaces; + int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces); + int n_secondTetFaces = p->mesh->getDownward(secondTet, 2, secondTetFaces); + int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, secondFace); + int fi2 = apf::findIn(secondTetFaces, n_secondTetFaces, secondFace); + PCU_ALWAYS_ASSERT(fi1 != -1 && fi2 != -1); + + // first tet opp vertex crd + apf::MeshEntity* firstTetOppVert = getTetOppVert( + p->mesh, firstTet, secondFace); + apf::Vector3 firstTetOppVertCrd; + p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd); + + // second tet opp vertex crd + apf::MeshEntity* secondTetOppVert = getTetOppVert( + p->mesh, secondTet, secondFace); + apf::Vector3 secondTetOppVertCrd; + p->mesh->getPoint(secondTetOppVert, 0, secondTetOppVertCrd); + + // normal to the face + apf::Downward edge_vertices; + p->mesh->getDownward(p->entity, 0, edge_vertices); + apf::Vector3 p0, p1, p2; + p->mesh->getPoint(edge_vertices[0], 0, p0); + p->mesh->getPoint(edge_vertices[1], 0, p1); + apf::MeshEntity* faceOppVert = getFaceOppVert( + p->mesh, secondFace, p->entity); + p->mesh->getPoint(faceOppVert, 0, p2); + + apf::Vector3 normal = apf::cross(p1-p0, p2-p0); + + // direction vectors from p0 to opp tet verts + apf::Vector3 vFirst = firstTetOppVertCrd - p0; + apf::Vector3 vLast = secondTetOppVertCrd - p0; + + if ((vFirst * normal > 0) && (vLast * normal < 0)) { + // reverse list of tets and faces + std::reverse(p->tets.begin(), p->tets.end()); + std::reverse(p->faces.begin(), p->faces.begin()); + } + } + } + else { + apf::MeshEntity* firstFace = p->faces[0]; + apf::MeshEntity* firstTet = p->tets[0]; + apf::MeshEntity* lastTet = p->tets[p->tets.size()-1]; + + apf::Downward firstTetFaces, lastTetFaces; + int n_firstTetFaces = p->mesh->getDownward(firstTet, 2, firstTetFaces); + int n_lastTetFaces = p->mesh->getDownward(lastTet, 2, lastTetFaces); + int fi1 = apf::findIn(firstTetFaces, n_firstTetFaces, firstFace); + int filast = apf::findIn(lastTetFaces, n_lastTetFaces, firstFace); + PCU_ALWAYS_ASSERT(fi1 != -1 && filast != -1); + + // first tet opp vertex crd + apf::MeshEntity* firstTetOppVert = getTetOppVert( + p->mesh, firstTet, firstFace); + apf::Vector3 firstTetOppVertCrd; + p->mesh->getPoint(firstTetOppVert, 0, firstTetOppVertCrd); + + // last tet opp vertex crd + apf::MeshEntity* lastTetOppVert = getTetOppVert( + p->mesh, lastTet, firstFace); + apf::Vector3 lastTetOppVertCrd; + p->mesh->getPoint(lastTetOppVert, 0, lastTetOppVertCrd); + + // normal to the face + apf::Downward edge_vertices; + p->mesh->getDownward(p->entity, 0, edge_vertices); + apf::Vector3 p0, p1, p2; + p->mesh->getPoint(edge_vertices[0], 0, p0); + p->mesh->getPoint(edge_vertices[1], 0, p1); + apf::MeshEntity* faceOppVert = getFaceOppVert( + p->mesh, firstFace, p->entity); + p->mesh->getPoint(faceOppVert, 0, p2); + + apf::Vector3 normal = apf::cross(p1-p0, p2-p0); + + // direction vectors from p0 to opp tet verts + apf::Vector3 vFirst = firstTetOppVertCrd - p0; + apf::Vector3 vLast = lastTetOppVertCrd - p0; + + if ((vFirst * normal > 0) && (vLast * normal < 0)) { + // reverse list of tets and faces + std::reverse(p->tets.begin(), p->tets.end()); + std::reverse(p->faces.begin(), p->faces.begin()); + } + } +} + +static void runErm(EdgePatch* ep) +{ + getOrderedTetsandFaces(ep->mesh, ep->entity, ep->tets, ep->faces); + //getClockwiseTetsandFaces(ep); + assembleEdgePatchLHS(ep); + assembleEdgePatchRHS(ep); + mth::solveFromQR(ep->qr.Q, ep->qr.R, ep->b, ep->x); + + if (!ep->isOnBdry) { // solve At*mu = g for g + mth::Vector temp(ep->tets.size()); + mth::multiply(ep->At, ep->x, temp); + for (size_t i = 0; i < ep->tets.size(); i++) { + ep->x(i) = temp(i); + } + } + + int nf = ep->faces.size(); + for(int i = 0; i < nf; i++) { + apf::MeshEntity* face = ep->faces[i]; + + apf::Downward e; + int ned = ep->mesh->getDownward(face, 1, e); + int ei = apf::findIn(e, ned, ep->entity); + PCU_ALWAYS_ASSERT(ned == 3 && ei != -1); + + double components[3]; + apf::getComponents(ep->equilibration->g, face, 0, components); + components[ei] = ep->x(i); + apf::setComponents(ep->equilibration->g, face, 0, components); + } +} + + +class EdgePatchOp : public apf::CavityOp +{ +public: + EdgePatchOp(Equilibration* eq): + apf::CavityOp(eq->mesh) + { + setupEdgePatch(&edgePatch, eq); + } + virtual Outcome setEntity(apf::MeshEntity* e) + { + if (edgePatch.mesh->hasTag(e, edgePatch.equilibration->tag)) + return SKIP; + startEdgePatch(&edgePatch, e); + if ( ! buildEdgePatch(&edgePatch, this)) + return REQUEST; + return OK; + } + virtual void apply() + { + runErm(&edgePatch); + int n = VISITED; + edgePatch.mesh->setIntTag( + edgePatch.entity, edgePatch.equilibration->tag, &n); + } + EdgePatch edgePatch; +}; + +apf::Field* equilibrateResiduals(apf::Field* f) +{ + apf::Field* g = createPackedField( + apf::getMesh(f), "g", 3, apf::getConstant(2) ); + Equilibration equilibration; + setupEquilibration(&equilibration, f, g); + EdgePatchOp op(&equilibration); + op.applyToDimension(1); // edges + + apf::MeshEntity* ent; + apf::MeshIterator* it = apf::getMesh(f)->begin(1); + while ((ent = apf::getMesh(f)->iterate(it))) { + apf::getMesh(f)->removeTag(ent, equilibration.tag); + } + apf::getMesh(f)->end(it); + apf::getMesh(f)->destroyTag(equilibration.tag); + + return g; +} + + +} \ No newline at end of file From 00bf25ea5cbc6e21613016b73969b08e7a528ffd Mon Sep 17 00:00:00 2001 From: Flynn McCallum <102677191+flagdanger@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:42:30 -0400 Subject: [PATCH 111/141] Delete config.sh --- config.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 config.sh diff --git a/config.sh b/config.sh deleted file mode 100644 index 21394a02a..000000000 --- a/config.sh +++ /dev/null @@ -1,6 +0,0 @@ -cmake -S . -B /lore/mccalf/build \ - -DCMAKE_C_COMPILER=mpicc \ - -DCMAKE_CXX_COMPILER=mpicxx \ - -DIS_TESTING=on \ - -DBUILD_EXES=on \ - -DMESHES=/users/mccalf/pcu-update/pumi-meshes From ea3b36ed9b87adfad0f611330358d61e0072fe83 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 29 Apr 2024 11:20:25 -0400 Subject: [PATCH 112/141] Warning fixes --- mds/apfMDS.cc | 2 +- mds/apfMDS.h | 2 +- parma/diffMC/parma_stop.h | 4 ++-- phasta/phCook.cc | 2 +- phasta/phiotimer.cc | 1 + phasta/phstream.cc | 1 + test/generate.cc | 1 + test/split.cc | 2 +- 8 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 53369737f..761614e65 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -1117,7 +1117,7 @@ Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, } Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, - int factor, pcu::PCU *PCUObj, pcu::PCU *oldPCU) + int factor, pcu::PCU *PCUObj) { m = expandMdsMesh(m, g, PCUObj->Peers() / factor, PCUObj); double t0 = pcu::Time(); diff --git a/mds/apfMDS.h b/mds/apfMDS.h index b0557ce43..56b287a58 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -117,7 +117,7 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder=false, bool copy void reorderMdsMesh(Mesh2* mesh, MeshTag* t = 0); Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor); -Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj, pcu::PCU *oldPCU = nullptr); +Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj); Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount); Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU); diff --git a/parma/diffMC/parma_stop.h b/parma/diffMC/parma_stop.h index daf7f2031..24439971e 100644 --- a/parma/diffMC/parma_stop.h +++ b/parma/diffMC/parma_stop.h @@ -8,12 +8,12 @@ namespace parma { class Stop { public: virtual ~Stop() {} - virtual bool stop(double imb, double maxImb, pcu::PCU *PCUObj)=0; + virtual bool stop(double imb, double maxImb, pcu::PCU*)=0; }; class Less : public Stop { public: ~Less() {} - bool stop(double imb, double maxImb, pcu::PCU *PCUObj) { + bool stop(double imb, double maxImb, pcu::PCU*) { return imb < maxImb; } }; diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 7dfbfdb8b..999246376 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -103,7 +103,7 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, }//end namespace namespace chef { - static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU *PCUObj) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { FILE* f = NULL; PHASTAIO_OPENTIME(f = pcu_group_open(path, false);) return f; diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index b1648a93d..4444b1eee 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -282,6 +282,7 @@ void phastaio_printStats(PCUHandle h) { void phastaio_initStats(PCUHandle h) { //removed checking if PCU is initiallized as it's a class invarient of apf::Mesh that it is + (void) h; #ifdef __INTEL_COMPILER phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(h); #endif diff --git a/phasta/phstream.cc b/phasta/phstream.cc index ce5b044f6..2fe9b8d0c 100644 --- a/phasta/phstream.cc +++ b/phasta/phstream.cc @@ -23,6 +23,7 @@ namespace { inline void printTime(const char* key, double t, pcu::PCU *pcu_obj) { (void) key; (void) t; + (void) pcu_obj; #if PHSTREAM_TIMERS_ON==1 if( isRankZero(pcu_obj) ) lion_eprint(1, "%s %f seconds\n", key, t); diff --git a/test/generate.cc b/test/generate.cc index f72ce3c43..91816d50f 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/test/split.cc b/test/split.cc index fe1f6925a..48d1ee7ba 100644 --- a/test/split.cc +++ b/test/split.cc @@ -94,7 +94,7 @@ int main(int argc, char** argv) //on the globalPCU if(m != nullptr) m->switchPCU(PCUObj.get()); delete groupedPCUObj; - m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get(), groupedPCUObj); + m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); From 769c432a922b14bfd51aec56174f4e3f57103eb3 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 2 May 2024 18:57:19 -0400 Subject: [PATCH 113/141] compiler errors with phLinks.cc fixed --- phasta/phLinks.cc | 6 +++--- test/viz.cc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/phasta/phLinks.cc b/phasta/phLinks.cc index 80d59e80f..73a2a0ff3 100644 --- a/phasta/phLinks.cc +++ b/phasta/phLinks.cc @@ -103,15 +103,15 @@ void getLinks(apf::Mesh* m, int dim, Links& links, BCs& bcs) apf::MeshEntity* v; while ((v = m->iterate(it))) { apf::ModelEntity* me = m->toModel(v); - shr->isDG = ph::isInterface(m->getModel(),(gmi_ent*) me,bcs.fields["DG interface"]); + shr.isDG = ph::isInterface(m->getModel(),(gmi_ent*) me,bcs.fields["DG interface"]); /* the alignment is such that the owner part's array follows the order of its vertex iterator traversal. The owner dictates the order to the other part by sending remote copies */ - if ( ! shr->isOwned(v)) + if ( ! shr.isOwned(v)) continue; apf::CopyArray remotes; - shr->getCopies(v, remotes); + shr.getCopies(v, remotes); for (size_t i = 0; i < remotes.getSize(); ++i) { /* in matching we may accumulate multiple occurrences of the same master in the outgoing links array diff --git a/test/viz.cc b/test/viz.cc index 28a6b5349..ce1a510d8 100644 --- a/test/viz.cc +++ b/test/viz.cc @@ -48,7 +48,7 @@ int main(int argc, char** argv) Visualization v; char output[128]; - sprintf(output,"%d",PCU_Comm_Self()); + sprintf(output,"%d",PCUObj.get()->Self()); std::string part_num(output); apf::MeshIterator* itr; From f96b7648c7ea0f936ce6e67e4ea1d2ce6f21b338 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 6 May 2024 10:39:17 -0400 Subject: [PATCH 114/141] .github fix --- .github/workflows/cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 39ed4a701..70c3f13b9 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -36,7 +36,7 @@ jobs: env: MPICH_CXX: ${{matrix.compiler.CXX}} MPICH_CC: ${{matrix.compiler.CC}} - run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j + run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j 1 - name: Test env: From 1e3d684d57c40cc1fb11385e4aa2ce2f3b92f505 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 6 May 2024 10:59:43 -0400 Subject: [PATCH 115/141] describe.cc getPeak() and getChunks() removed --- test/describe.cc | 57 ++---------------------------------------------- 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/test/describe.cc b/test/describe.cc index 95bf0d084..0fc79146f 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -18,37 +18,6 @@ #include #include -#ifdef __bgq__ -#include - -static double get_peak(pcu::PCU*) -{ - uint64_t heap; - Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap); - return heap; -} - -#elif defined (__linux__) - -static double get_peak(pcu::PCU*) -{ -#if defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2) - return mallinfo2().arena; -#elif defined(__GNUG__) - return mallinfo().arena; -#endif -} - -#else - -static double get_peak(pcu::PCU *PCUObj) -{ - if(!PCUObj->Self()) - printf("%s:%d: OS Not supported\n", __FILE__, __LINE__); - return(-1.0); -} - -#endif static void print_stats(const char* name, double value, pcu::PCU *PCUObj) @@ -63,27 +32,6 @@ static void print_stats(const char* name, double value, pcu::PCU *PCUObj) printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb); } -#if defined(__linux__) - -static double get_chunks(pcu::PCU*) -{ -#if defined(__GNUG__) && defined(PUMI_HAS_MALLINFO2) - struct mallinfo2 m = mallinfo2(); -#elif defined(__GNUG__) - struct mallinfo m = mallinfo(); -#endif - return m.uordblks + m.hblkhd; -} - -#else -static double get_chunks(pcu::PCU *PCUObj) -{ - if(!PCUObj->Self()) - printf("%s:%d: OS Not supported\n", __FILE__, __LINE__); - return(-1.0); -} -#endif - static void list_tags(apf::Mesh* m) { if (m->getPCU()->Self()) @@ -109,11 +57,10 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - print_stats("malloc used before", get_chunks(pcu_obj.get()), pcu_obj.get()); + print_stats("kernal used before", pcu::GetMem(), pcu_obj.get()); apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); m->verify(); - print_stats("kernel heap", get_peak(pcu_obj.get()), pcu_obj.get()); - print_stats("malloc used", get_chunks(pcu_obj.get()), pcu_obj.get()); + print_stats("kernel heap", pcu::GetMem(), pcu_obj.get()); Parma_PrintPtnStats(m, ""); list_tags(m); m->destroyNative(); From e33b48d52769a1835fae3198108eea4d4f5cf295 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 3 Jun 2024 13:28:38 -0400 Subject: [PATCH 116/141] removed last uses of old pcu, removed pcu.cc and .h, got rid of default nullptr PCUObj arguments --- apf/apfMesh.cc | 9 +- apf/apfMesh.h | 3 +- apf/apfMesh2.cc | 61 +- apf/apfMesh2.h | 2 +- mds/apfMDS.cc | 140 +---- mds/apfMDS.h | 20 +- mds/mdsANSYS.cc | 14 +- mds/mdsGmsh.cc | 12 +- mds/mdsUgrid.cc | 12 +- mds/mds_apf.c | 22 +- mds/mds_apf.h | 21 +- mds/mds_net.c | 44 +- mds/mds_net.h | 19 +- mds/mds_order.c | 9 +- mds/mds_smb.c | 40 +- .../maximalIndependentSet/mersenne_twister.cc | 1 - parma/group/parma_group.cc | 4 +- pcu/CMakeLists.txt | 4 +- pcu/PCU.h | 133 ---- pcu/PCUObj.cc | 2 +- pcu/pcu.cc | 581 ------------------ pcu/pcu_io.c | 25 +- pcu/pcu_io.h | 9 +- phasta/ph.h | 2 +- phasta/phCook.cc | 8 +- phasta/phIO.c | 6 +- phasta/phIO.h | 4 +- phasta/phRestart.cc | 6 +- pumi/pumi.h | 2 +- pumi/pumi_mesh.cc | 25 +- sam/samElementCount.cc | 1 - test/ph_adapt.cc | 4 +- test/pumi.cc | 2 +- test/ugrid.cc | 1 - test/ugridptnstats.cc | 2 +- 35 files changed, 107 insertions(+), 1143 deletions(-) delete mode 100644 pcu/PCU.h delete mode 100644 pcu/pcu.cc diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 1338e0ab8..125dd825f 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include "apfCoordData.h" #include "apfVectorField.h" @@ -167,13 +166,7 @@ void Mesh::init(FieldShape* s, pcu::PCU *PCUObj) baseP->init("coordinates",this,s,data); data->init(baseP); hasFrozenFields = false; - if(PCUObj != nullptr){ - pcu_ = PCUObj; - } else { - pcu_ = pcu::PCU_GetGlobal(); - PCU_ALWAYS_ASSERT_VERBOSE(pcu_ != nullptr, "PCU must be initialized\n"); - } - + pcu_ = PCUObj; } Mesh::~Mesh() diff --git a/apf/apfMesh.h b/apf/apfMesh.h index c56ba26c5..5272263ec 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -14,7 +14,6 @@ #include #include #include -#include #include #include "apfVector.h" #include "apfDynamicArray.h" @@ -109,7 +108,7 @@ class Mesh \param s the field distribution of the coordinate field, apf::getLagrange(1) is a good default */ - void init(FieldShape* s, pcu::PCU *PCUObj = nullptr); + void init(FieldShape* s, pcu::PCU *PCUObj); /** \brief destroy the base class structures. \details this does not destroy the underlying data structure, use apf::Mesh::destroyNative for that. diff --git a/apf/apfMesh2.cc b/apf/apfMesh2.cc index 329867b63..dfbca80aa 100644 --- a/apf/apfMesh2.cc +++ b/apf/apfMesh2.cc @@ -207,25 +207,13 @@ static void packTagClone(Mesh2* m, MeshTag* t, int to, pcu::PCU *PCUObj) { std::string name; name = m->getTagName(t); - if(PCUObj != nullptr){ - packString(name, to, PCUObj); - } else { - packString(name, to, m->getPCU()); - } + packString(name, to, PCUObj); int type; type = m->getTagType(t); - if(PCUObj != nullptr){ - PCUObj->Pack(to, type); - } else { - m->getPCU()->Pack(to, type); - } + PCUObj->Pack(to, type); int size; size = m->getTagSize(t); - if(PCUObj != nullptr){ - PCUObj->Pack(to, size); - } else { - m->getPCU()->Pack(to, size); - } + PCUObj->Pack(to, size); } static MeshTag* unpackTagClone(Mesh2* m) @@ -249,11 +237,7 @@ static void packTagClones(Mesh2* m, int to, pcu::PCU *PCUObj) DynamicArray tags; m->getTags(tags); int n = tags.getSize(); - if(PCUObj != nullptr){ - PCUObj->Pack(to, n); - } else { - m->getPCU()->Pack(to, n); - } + PCUObj->Pack(to, n); /* warning! this loop goes backward to cater to MDS implementation-specific behavior. @@ -273,32 +257,15 @@ static void unpackTagClones(Mesh2* m) static void packFieldClone(Field* f, int to, pcu::PCU *PCUObj) { std::string name = f->getName(); - if(PCUObj != nullptr){ - packString(name, to, PCUObj); - } else { - packString(name, to, f->getMesh()->getPCU()); - } - + packString(name, to, PCUObj); int valueType = f->getValueType(); - if(PCUObj != nullptr){ - PCUObj->Pack(to, valueType); - } else { - f->getMesh()->getPCU()->Pack(to, valueType); - } + PCUObj->Pack(to, valueType); int components = f->countComponents(); - if(PCUObj != nullptr){ - PCUObj->Pack(to, components); - } else { - f->getMesh()->getPCU()->Pack(to, components); - } + PCUObj->Pack(to, components); std::string shapeName = f->getShape()->getName(); - if(PCUObj != nullptr){ - packString(shapeName, to, PCUObj); - } else { - packString(shapeName, to, f->getMesh()->getPCU()); - } + packString(shapeName, to, PCUObj); /* warning! this only supports tag-stored fields */ } @@ -320,11 +287,7 @@ static Field* unpackFieldClone(Mesh2* m) static void packFieldClones(Mesh2* m, int to, pcu::PCU *PCUObj) { int n = m->countFields(); - if(PCUObj != nullptr){ - PCUObj->Pack(to, n); - } else { - m->getPCU()->Pack(to, n); - } + PCUObj->Pack(to, n); for (int i = 0; i < n; ++i) packFieldClone(m->getField(i), to, PCUObj); @@ -341,11 +304,7 @@ static void unpackFieldClones(Mesh2* m) static void packMeshShape(Mesh2* m, int to, pcu::PCU *PCUObj) { std::string shapeName = m->getShape()->getName(); - if(PCUObj != nullptr){ - packString(shapeName, to, PCUObj); - } else { - packString(shapeName, to, m->getPCU()); - } + packString(shapeName, to, PCUObj); } static void unpackMeshShape(Mesh2* m) diff --git a/apf/apfMesh2.h b/apf/apfMesh2.h index c11c0a534..2dda18e9b 100644 --- a/apf/apfMesh2.h +++ b/apf/apfMesh2.h @@ -199,7 +199,7 @@ void stitchMesh(Mesh2* m); /** \brief removes all entities and fields. */ void clear(Mesh2* m); -void packDataClone(Mesh2* m, int to, pcu::PCU *PCUObj = nullptr); +void packDataClone(Mesh2* m, int to, pcu::PCU *PCUObj); void unpackDataClone(Mesh2* m); // common functions for migration/ghosting/distribution diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 761614e65..77fe67180 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -8,7 +8,6 @@ *******************************************************************************/ -#include #include #include #include @@ -169,13 +168,9 @@ class MeshMDS : public Mesh2 isMatched = false; ownsModel = false; } - MeshMDS(gmi_model* m, int d, bool isMatched_, pcu::PCU *PCUObj = nullptr) + MeshMDS(gmi_model* m, int d, bool isMatched_, pcu::PCU *PCUObj) { - if(PCUObj != nullptr){ - init(apf::getLagrange(1), PCUObj); - } else { - init(apf::getLagrange(1)); - } + init(apf::getLagrange(1), PCUObj); mds_id cap[MDS_TYPES] = {}; mesh = mds_apf_create(m, d, cap); isMatched = isMatched_; @@ -184,11 +179,7 @@ class MeshMDS : public Mesh2 MeshMDS(gmi_model* m, Mesh* from, apf::MeshEntity** nodes, apf::MeshEntity** elems, bool copy_data=true) { - if(from->getPCU() != nullptr){ - init(apf::getLagrange(1), from->getPCU()); - } else { - init(apf::getLagrange(1)); - } + init(apf::getLagrange(1), from->getPCU()); mds_id cap[MDS_TYPES]; cap[MDS_VERTEX] = from->count(0); cap[MDS_EDGE] = from->count(1); @@ -205,14 +196,10 @@ class MeshMDS : public Mesh2 apf::convert(from, this, nodes, elems, copy_data); } - MeshMDS(gmi_model* m, const char* pathname, pcu::PCU *PCUObj = nullptr) + MeshMDS(gmi_model* m, const char* pathname, pcu::PCU *PCUObj) { - if(PCUObj != nullptr){ - init(apf::getLagrange(1), PCUObj); - } else { - init(apf::getLagrange(1)); - } - mesh = mds_read_smb2(this->getPCU()->GetCHandle(), m, pathname, 0, this); + init(apf::getLagrange(1), PCUObj); + mesh = mds_read_smb(this->getPCU()->GetCHandle(), m, pathname, 0, this); isMatched = this->getPCU()->Or(!mds_net_empty(&mesh->matches)); ownsModel = true; } @@ -598,7 +585,7 @@ class MeshMDS : public Mesh2 void writeNative(const char* fileName) { double t0 = pcu::Time(); - mesh = mds_write_smb2(this->getPCU()->GetCHandle(), mesh, fileName, 0, this); + mesh = mds_write_smb(this->getPCU()->GetCHandle(), mesh, fileName, 0, this); double t1 = pcu::Time(); if (!this->getPCU()->Self()) lion_oprint(1,"mesh %s written in %f seconds\n", fileName, t1 - t0); @@ -794,11 +781,7 @@ class MeshMDS : public Mesh2 Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj) { Mesh2* m; - if(PCUObj != nullptr){ - m = new MeshMDS(model, dim, isMatched, PCUObj); - } else { - m = new MeshMDS(model, dim, isMatched); - } + m = new MeshMDS(model, dim, isMatched, PCUObj); initResidence(m, dim); return m; } @@ -943,11 +926,7 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder, bool copy_data) Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { Mesh2* m; - if(PCUObj != nullptr){ - m = new MeshMDS(model, meshfile, PCUObj); - } else { - m = new MeshMDS(model, meshfile); - } + m = new MeshMDS(model, meshfile, PCUObj); return m; } @@ -966,23 +945,6 @@ Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) return m; } - -Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile) -{ - double t0 = pcu::Time(); - Mesh2* m = new MeshMDS(model, meshfile); - initResidence(m, m->getDimension()); - stitchMesh(m); - m->acceptChanges(); - - if (!m->getPCU()->Self()) - lion_oprint(1,"mesh %s loaded in %f seconds\n", meshfile, pcu::Time() - t0); - printStats(m); - warnAboutEmptyParts(m); - return m; -} - - Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj) { double t0 = pcu::Time(); @@ -994,19 +956,6 @@ Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj return loadMdsMesh(model, meshfile, PCUObj); } -Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile) -{ - double t0 = pcu::Time(); - static gmi_model* model; - model = gmi_load(modelfile); - if (!PCU_Comm_Self()) - lion_oprint(1,"model %s loaded in %f seconds\n", modelfile, pcu::Time() - t0); - - return loadMdsMesh(model, meshfile); -} - - - void reorderMdsMesh(Mesh2* mesh, MeshTag* t) { double t0 = pcu::Time(); @@ -1018,7 +967,7 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) } else { vert_nums = mds_number_verts_bfs(m->mesh); } - m->mesh = mds_reorder2(mesh->getPCU()->GetCHandle(), m->mesh, 0, vert_nums); + m->mesh = mds_reorder(mesh->getPCU()->GetCHandle(), m->mesh, 0, vert_nums); if (!mesh->getPCU()->Self()) lion_oprint(1,"mesh reordered in %f seconds\n", pcu::Time()-t0); } @@ -1061,61 +1010,6 @@ Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expan return m; } - -Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) -{ - double t0 = pcu::Time(); - int self = PCU_Comm_Self(); - int outputPartCount = PCU_Comm_Peers(); - apf::Expand expand(inputPartCount, outputPartCount); - apf::Contract contract(inputPartCount, outputPartCount); - bool isOriginal = contract.isValid(self); - int dim; - bool isMatched; - PCU_Comm_Begin(); - if (isOriginal) { - PCU_ALWAYS_ASSERT(m != 0); - dim = m->getDimension(); - isMatched = m->hasMatching(); - for (int i = self + 1; i < outputPartCount && !contract.isValid(i); ++i) { - PCU_COMM_PACK(i, dim); - PCU_COMM_PACK(i, isMatched); - packDataClone(m, i); - } - } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(dim); - PCU_COMM_UNPACK(isMatched); - m = makeEmptyMdsMesh(g, dim, isMatched); - unpackDataClone(m); - } - PCU_ALWAYS_ASSERT(m != 0); - apf::remapPartition(m, expand); - double t1 = pcu::Time(); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh expanded from %d to %d parts in %f seconds\n", - inputPartCount, outputPartCount, t1 - t0); - return m; -} - -Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, - int factor) -{ - m = expandMdsMesh(m, g, PCU_Comm_Peers() / factor); - double t0 = pcu::Time(); - if (PCU_Comm_Self() % factor != 0) - plan = new apf::Migration(m, m->findTag("apf_migrate")); - m->migrate(plan); - double t1 = pcu::Time(); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh migrated from %d to %d in %f seconds\n", - PCU_Comm_Peers() / factor, - PCU_Comm_Peers(), - t1 - t0); - return m; -} - Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj) { @@ -1138,13 +1032,13 @@ bool alignMdsMatches(Mesh2* in) if (!in->hasMatching()) return false; MeshMDS* m = static_cast(in); - return mds_align_matches2(in->getPCU()->GetCHandle(), m->mesh); + return mds_align_matches(in->getPCU()->GetCHandle(), m->mesh); } bool alignMdsRemotes(Mesh2* in) { MeshMDS* m = static_cast(in); - return mds_align_remotes2(in->getPCU()->GetCHandle(), m->mesh); + return mds_align_remotes(in->getPCU()->GetCHandle(), m->mesh); } void deriveMdsModel(Mesh2* in) @@ -1413,12 +1307,8 @@ void hackMdsAdjacency(Mesh2* in, MeshEntity* up, int i, MeshEntity* down) Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { MeshMDS* m = new MeshMDS(); - if(PCUObj != nullptr){ - m->init(apf::getLagrange(1), PCUObj); - } else { - m->init(apf::getLagrange(1)); - } - m->mesh = mds_read_smb2(m->getPCU()->GetCHandle(), model, meshfile, 1, m); + m->init(apf::getLagrange(1), PCUObj); + m->mesh = mds_read_smb(m->getPCU()->GetCHandle(), model, meshfile, 1, m); m->isMatched = false; m->ownsModel = true; initResidence(m, m->getDimension()); @@ -1428,7 +1318,7 @@ Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) void writeMdsPart(Mesh2* in, const char* meshfile) { MeshMDS* m = static_cast(in); - m->mesh = mds_write_smb2(m->getPCU()->GetCHandle(), m->mesh, meshfile, 1, m); + m->mesh = mds_write_smb(m->getPCU()->GetCHandle(), m->mesh, meshfile, 1, m); } diff --git a/mds/apfMDS.h b/mds/apfMDS.h index 56b287a58..7a5aa1477 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -62,7 +62,7 @@ typedef struct PCUHandle PCUHandle; \param dim the eventual mesh dimension. MDS needs to allocate arrays based on this before users add entities. \param isMatched whether or not there will be matched entities */ -Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj = nullptr); +Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj); /** \brief load an MDS mesh and model from file \param modelfile will be passed to gmi_load to get the model @@ -72,7 +72,6 @@ Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCU Parasolid, and ACIS models */ Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj); -Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile); /** \brief load an MDS mesh from files \param model the geometric model interface @@ -88,10 +87,9 @@ Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile); Calling apf::Mesh::writeNative on the resulting object will do the same in reverse. */ Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); -Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile); // make a serial mesh on all processes - no pmodel & remote link setup -Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj = nullptr); +Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); /** \brief create an MDS mesh from an existing mesh \param from the mesh to copy @@ -116,10 +114,8 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder=false, bool copy there are no gaps in the MDS arrays after this */ void reorderMdsMesh(Mesh2* mesh, MeshTag* t = 0); -Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor); Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj); -Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount); Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU); /** \brief align the downward adjacencies of matched entities */ bool alignMdsMatches(Mesh2* in); @@ -218,14 +214,14 @@ Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCM int gmshMajorVersion(const char* filename); -Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj = nullptr); +Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj); -Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename, pcu::PCU *PCUObj = nullptr); +Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename, pcu::PCU *PCUObj); -Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj = nullptr); +Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj); void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile, - const double elmWeights[], pcu::PCU *PCUObj = nullptr); + const double elmWeights[], pcu::PCU *PCUObj); /** \brief load an MDS mesh from ANSYS .node and .elem files \details this call takes two filenames, one @@ -238,13 +234,13 @@ void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile currently, ANSYS element types SOLID72 and SOLID92 are supported, which become linear and quadratic tetrahedra, respectively. */ -Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PCUObj = nullptr); +Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PCUObj); void disownMdsModel(Mesh2* in); void setMdsMatching(Mesh2* in, bool has); -Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj = nullptr); +Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); void writeMdsPart(Mesh2* m, const char* meshfile); } diff --git a/mds/mdsANSYS.cc b/mds/mdsANSYS.cc index 894d96953..9ee145f76 100644 --- a/mds/mdsANSYS.cc +++ b/mds/mdsANSYS.cc @@ -90,7 +90,7 @@ static void parseNodes(const char* nodefile, Nodes& nodes) typedef std::map Vertices; -static Mesh2* parseElems(const char* elemfile, Nodes& nodes, pcu::PCU *PCUObj = nullptr) +static Mesh2* parseElems(const char* elemfile, Nodes& nodes, pcu::PCU *PCUObj) { Mesh2* m = 0; std::ifstream f(elemfile); @@ -107,11 +107,7 @@ static Mesh2* parseElems(const char* elemfile, Nodes& nodes, pcu::PCU *PCUObj = apf::Numbering* enumbers = 0; while (parseElem(f, en, type, id, shape)) { if (!m) { - if(PCUObj != nullptr){ - m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false, PCUObj); - } else { - m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false); - } + m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false, PCUObj); if (shape != m->getShape()) changeMeshShape(m, shape, false); @@ -161,11 +157,7 @@ Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PC Nodes nodes; parseNodes(nodefile, nodes); Mesh2* m; - if(PCUObj != nullptr){ - m = parseElems(elemfile, nodes, PCUObj); - } else { - m = parseElems(elemfile, nodes); - } + m = parseElems(elemfile, nodes, PCUObj); m->acceptChanges(); deriveMdsModel(m); return m; diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc index 99ca5af24..2406467eb 100644 --- a/mds/mdsGmsh.cc +++ b/mds/mdsGmsh.cc @@ -462,11 +462,7 @@ void gmshFindDmg(const char* fnameDmg, const char* filename) Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { Mesh2* m; - if(PCUObj != nullptr){ - m = makeEmptyMdsMesh(g, 0, false, PCUObj); - } else { - m = makeEmptyMdsMesh(g, 0, false); - } + m = makeEmptyMdsMesh(g, 0, false, PCUObj); readGmsh(m, filename); return m; } @@ -475,11 +471,7 @@ Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename, pcu::PCU *P { gmshFindDmg(fnameDmg, filename); // new function that scans $Entities and writes a dmg Mesh2* m; - if(PCUObj != nullptr){ - m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); - } else { - m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false); - } + m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); readGmsh(m, filename); return m; } diff --git a/mds/mdsUgrid.cc b/mds/mdsUgrid.cc index f7fe31545..df4dc5ac4 100644 --- a/mds/mdsUgrid.cc +++ b/mds/mdsUgrid.cc @@ -664,11 +664,7 @@ namespace apf { Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { Mesh2* m; - if(PCUObj != nullptr){ - m = makeEmptyMdsMesh(g, 0, false, PCUObj); - } else { - m = makeEmptyMdsMesh(g, 0, false); - } + m = makeEmptyMdsMesh(g, 0, false, PCUObj); header hdr; Reader r; initReader(&r, m, filename); @@ -691,11 +687,7 @@ namespace apf { void printUgridPtnStats(gmi_model* g, const char* ufile, const char* vtxptn, const double elmWeights[], pcu::PCU *PCUObj) { Mesh2* m; - if(PCUObj != nullptr){ - m = makeEmptyMdsMesh(g, 0, false, PCUObj); - } else { - m = makeEmptyMdsMesh(g, 0, false); - } + m = makeEmptyMdsMesh(g, 0, false, PCUObj); apf::changeMdsDimension(m, 3); printPtnStats(m, ufile, vtxptn, elmWeights); m->destroyNative(); diff --git a/mds/mds_apf.c b/mds/mds_apf.c index 915f25d05..1a7903016 100644 --- a/mds/mds_apf.c +++ b/mds/mds_apf.c @@ -286,36 +286,20 @@ static int align_copies(PCUHandle h, struct mds_net* net, struct mds* m) return PCU_Or2(h, did_change); } -int mds_align_matches(struct mds_apf* m) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_align_matches2(h, m); -} -int mds_align_matches2(PCUHandle h, struct mds_apf* m) +int mds_align_matches(PCUHandle h, struct mds_apf* m) { return align_copies(h, &m->matches, &m->mds); } // seol -int mds_align_ghosts(struct mds_apf* m) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_align_ghosts2(h, m); -} - -int mds_align_ghosts2(PCUHandle h, struct mds_apf* m) +int mds_align_ghosts(PCUHandle h, struct mds_apf* m) { return align_copies(h, &m->ghosts, &m->mds); } -int mds_align_remotes(struct mds_apf* m) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_align_remotes2(h, m); -} -int mds_align_remotes2(PCUHandle h, struct mds_apf* m) +int mds_align_remotes(PCUHandle h, struct mds_apf* m) { return align_copies(h, &m->remotes, &m->mds); } diff --git a/mds/mds_apf.h b/mds/mds_apf.h index a453b9a0c..38bebf423 100644 --- a/mds/mds_apf.h +++ b/mds/mds_apf.h @@ -57,33 +57,24 @@ void* mds_get_part(struct mds_apf* m, mds_id e); void mds_set_part(struct mds_apf* m, mds_id e, void* p); struct mds_tag* mds_number_verts_bfs(struct mds_apf* m); -struct mds_apf* mds_reorder(struct mds_apf* m, int ignore_peers, - struct mds_tag* vert_numbers); -struct mds_apf* mds_reorder2(PCUHandle h, struct mds_apf* m, int ignore_peers, +struct mds_apf* mds_reorder(PCUHandle h, struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers); struct gmi_ent* mds_find_model(struct mds_apf* m, int dim, int id); int mds_model_dim(struct mds_apf* m, struct gmi_ent* model); int mds_model_id(struct mds_apf* m, struct gmi_ent* model); -struct mds_apf* mds_read_smb(struct gmi_model* model, const char* pathname, - int ignore_peers, void* apf_mesh); -struct mds_apf* mds_read_smb2(PCUHandle h, struct gmi_model* model, const char* pathname, - int ignore_peers, void* apf_mesh); -struct mds_apf* mds_write_smb(struct mds_apf* m, const char* pathname, +struct mds_apf* mds_read_smb(PCUHandle h, struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh); -struct mds_apf* mds_write_smb2(PCUHandle h, struct mds_apf* m, const char* pathname, +struct mds_apf* mds_write_smb(PCUHandle h, struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh); void mds_verify(struct mds_apf* m); void mds_verify_residence(struct mds_apf* m, mds_id e); -int mds_align_matches(struct mds_apf* m); -int mds_align_matches2(PCUHandle h, struct mds_apf* m); -int mds_align_ghosts(struct mds_apf* m); -int mds_align_ghosts2(PCUHandle h, struct mds_apf* m); -int mds_align_remotes(struct mds_apf* m); -int mds_align_remotes2(PCUHandle h, struct mds_apf* m); +int mds_align_matches(PCUHandle h, struct mds_apf* m); +int mds_align_ghosts(PCUHandle h, struct mds_apf* m); +int mds_align_remotes(PCUHandle h, struct mds_apf* m); void mds_derive_model(struct mds_apf* m); void mds_update_model_for_entity(struct mds_apf* m, mds_id e, diff --git a/mds/mds_net.c b/mds/mds_net.c index 8aa693d93..83cb83271 100644 --- a/mds/mds_net.c +++ b/mds/mds_net.c @@ -222,14 +222,7 @@ static void recv_links(PCUHandle h, struct mds_links* ln) ln->l[pi][i] = mds_index(tmp[i]); } -void mds_get_type_links(struct mds_net* net, struct mds* m, - int t, struct mds_links* ln) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_get_type_links2(h, net, m, t, ln); -} - -void mds_get_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_get_type_links(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; @@ -251,14 +244,7 @@ void mds_get_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, recv_links(h, ln); } -void mds_set_type_links(struct mds_net* net, struct mds* m, - int t, struct mds_links* ln) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_set_type_links2(h, net, m, t, ln); -} - -void mds_set_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_type_links(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; @@ -328,14 +314,7 @@ static void take_local_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) } } -void mds_get_local_matches(struct mds_net* net, struct mds* m, - int t, struct mds_links* ln) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_get_local_matches2(h, net, m, t, ln); -} - -void mds_get_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_get_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; @@ -352,14 +331,7 @@ void mds_get_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, for_type_net(h, net, m, t, take_local_link, ln); } -void mds_set_local_matches(struct mds_net* net, struct mds* m, - int t, struct mds_links* ln) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_set_local_matches2(h, net, m, t, ln); -} - -void mds_set_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; @@ -384,13 +356,7 @@ void mds_set_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, } } -void mds_free_local_links(struct mds_links* ln) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_free_local_links2(h, ln); -} - -void mds_free_local_links2(PCUHandle h, struct mds_links* ln) +void mds_free_local_links(PCUHandle h, struct mds_links* ln) { int self, other; self = find_peer(ln, PCU_Comm_Self2(h)); diff --git a/mds/mds_net.h b/mds/mds_net.h index 5cde35428..a1e6da1e8 100644 --- a/mds/mds_net.h +++ b/mds/mds_net.h @@ -52,27 +52,18 @@ void mds_grow_net( void mds_add_copy(struct mds_net* net, struct mds* m, mds_id e, struct mds_copy c); -void mds_get_type_links(struct mds_net* net, struct mds* m, +void mds_get_type_links(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_get_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, - int t, struct mds_links* ln); -void mds_set_type_links(struct mds_net* net, struct mds* m, - int t, struct mds_links* ln); -void mds_set_type_links2(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_type_links(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); void mds_free_links(struct mds_links* ln); int mds_net_empty(struct mds_net* net); -void mds_get_local_matches(struct mds_net* net, struct mds* m, - int t, struct mds_links* ln); -void mds_get_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, - int t, struct mds_links* ln); -void mds_set_local_matches(struct mds_net* net, struct mds* m, +void mds_get_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_set_local_matches2(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_free_local_links(struct mds_links* ln); -void mds_free_local_links2(PCUHandle h, struct mds_links* ln); +void mds_free_local_links(PCUHandle h, struct mds_links* ln); #endif diff --git a/mds/mds_order.c b/mds/mds_order.c index 5f5034550..b19c3a389 100644 --- a/mds/mds_order.c +++ b/mds/mds_order.c @@ -406,14 +406,7 @@ static struct mds_apf* rebuild( return m2; } -struct mds_apf* mds_reorder(struct mds_apf* m, int ignore_peers, - struct mds_tag* vert_numbers) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_reorder2(h, m, ignore_peers, vert_numbers); -} - -struct mds_apf* mds_reorder2(PCUHandle h, struct mds_apf* m, int ignore_peers, +struct mds_apf* mds_reorder(PCUHandle h, struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers) { struct mds_tag* tag; diff --git a/mds/mds_smb.c b/mds/mds_smb.c index def9d9fd9..edc6dc82d 100644 --- a/mds/mds_smb.c +++ b/mds/mds_smb.c @@ -218,7 +218,7 @@ static void read_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, struct mds_links ln = MDS_LINKS_INIT; read_links(f, &ln); if (!ignore_peers) - mds_set_type_links2(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); + mds_set_type_links(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); mds_free_links(&ln); } @@ -227,7 +227,7 @@ static void write_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, { struct mds_links ln = MDS_LINKS_INIT; if (!ignore_peers) - mds_get_type_links2(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); + mds_get_type_links(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); write_links(f, &ln); mds_free_links(&ln); } @@ -516,10 +516,10 @@ static void read_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m struct mds_links ln = MDS_LINKS_INIT; read_links(f, &ln); if (!ignore_peers) - mds_set_local_matches2(h, &m->matches, &m->mds, t, &ln); - mds_free_local_links2(h, &ln); + mds_set_local_matches(h, &m->matches, &m->mds, t, &ln); + mds_free_local_links(h, &ln); if (!ignore_peers) - mds_set_type_links2(h, &m->matches, &m->mds, t, &ln); + mds_set_type_links(h, &m->matches, &m->mds, t, &ln); mds_free_links(&ln); } @@ -528,8 +528,8 @@ static void write_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* { struct mds_links ln = MDS_LINKS_INIT; if (!ignore_peers) { - mds_get_type_links2(h, &m->matches, &m->mds, t, &ln); - mds_get_local_matches2(h, &m->matches, &m->mds, t, &ln); + mds_get_type_links(h, &m->matches, &m->mds, t, &ln); + mds_get_local_matches(h, &m->matches, &m->mds, t, &ln); } write_links(f, &ln); mds_free_links(&ln); @@ -571,7 +571,7 @@ static struct mds_apf* read_smb(PCUHandle h, struct gmi_model* model, const char int i; unsigned tmp; unsigned pi, pj; - f = pcu_fopen2(h, filename, 0, zip); + f = pcu_fopen(h, filename, 0, zip); PCU_ALWAYS_ASSERT(f); read_header(h, f, &version, &dim, ignore_peers); pcu_read_unsigneds(f, n, SMB_TYPES); @@ -620,7 +620,7 @@ static void write_smb(PCUHandle h, struct mds_apf* m, const char* filename, struct pcu_file* f; unsigned n[SMB_TYPES] = {0}; int i; - f = pcu_fopen2(h, filename, 1, zip); + f = pcu_fopen(h, filename, 1, zip); PCU_ALWAYS_ASSERT(f); write_header(h, f, m->mds.d, ignore_peers); for (i = 0; i < MDS_TYPES; ++i) @@ -731,14 +731,7 @@ static char* handle_path(PCUHandle h, const char* in, int is_write, int* zip, return path; } -struct mds_apf* mds_read_smb(struct gmi_model* model, const char* pathname, - int ignore_peers, void* apf_mesh) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_read_smb2(h, model, pathname, ignore_peers, apf_mesh); -} - -struct mds_apf* mds_read_smb2(PCUHandle h, struct gmi_model* model, const char* pathname, +struct mds_apf* mds_read_smb(PCUHandle h, struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh) { char* filename; @@ -759,14 +752,7 @@ static int is_compact(struct mds_apf* m) return 1; } -struct mds_apf* mds_write_smb(struct mds_apf* m, const char* pathname, - int ignore_peers, void* apf_mesh) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return mds_write_smb2(h, m, pathname, ignore_peers, apf_mesh); -} - -struct mds_apf* mds_write_smb2(PCUHandle h, struct mds_apf* m, const char* pathname, +struct mds_apf* mds_write_smb(PCUHandle h, struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh) { const char* reorderWarning ="MDS: reordering before writing smb files\n"; @@ -774,11 +760,11 @@ struct mds_apf* mds_write_smb2(PCUHandle h, struct mds_apf* m, const char* pathn int zip; if (ignore_peers && (!is_compact(m))) { if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); - m = mds_reorder2(h, m, 1, mds_number_verts_bfs(m)); + m = mds_reorder(h, m, 1, mds_number_verts_bfs(m)); } if ((!ignore_peers) && PCU_Or2(h, !is_compact(m))) { if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); - m = mds_reorder2(h, m, 0, mds_number_verts_bfs(m)); + m = mds_reorder(h, m, 0, mds_number_verts_bfs(m)); } filename = handle_path(h, pathname, 1, &zip, ignore_peers); write_smb(h, m, filename, zip, ignore_peers, apf_mesh); diff --git a/parma/diffMC/maximalIndependentSet/mersenne_twister.cc b/parma/diffMC/maximalIndependentSet/mersenne_twister.cc index 25172da7a..8938e2cff 100644 --- a/parma/diffMC/maximalIndependentSet/mersenne_twister.cc +++ b/parma/diffMC/maximalIndependentSet/mersenne_twister.cc @@ -1,5 +1,4 @@ #include "mersenne_twister.h" -#include "PCU.h" #include #include #include diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index ba43cb1c5..799428082 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -37,7 +37,7 @@ static void runInGroups( { int self; pcu::PCU* expandedPCU; - if(m != nullptr){ + if(PCUObj == nullptr){ self = m->getPCU()->Self(); expandedPCU = m->getPCU(); } else { @@ -130,7 +130,7 @@ void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu } apf::Modulo inMap(factor); apf::Divide groupMap(factor); - if(m != nullptr){ + if(PCUObj == nullptr){ apf::Unmodulo outMap(m->getPCU()->Self(), factor); runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); } else { diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 044f83c31..d91114514 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,7 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.cc + #pcu.cc pcu2.cc pcu_aa.c pcu_coll.c @@ -27,7 +27,7 @@ set(SOURCES # Package headers set(HEADERS - PCU.h + #PCU.h PCU2.h pcu_io.h pcu_util.h diff --git a/pcu/PCU.h b/pcu/PCU.h deleted file mode 100644 index 8b3e2c866..000000000 --- a/pcu/PCU.h +++ /dev/null @@ -1,133 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -#ifndef PCU_H -#define PCU_H -#include "pcu_defines.h" -#include - -#ifdef __cplusplus -#include -#include -extern "C" { -#else -#include -#include -#include -#endif - -/*library init/finalize*/ -int PCU_Comm_Init(void); -int PCU_Comm_Free(void); - -/*rank/size functions*/ -int PCU_Comm_Self(void); -int PCU_Comm_Peers(void); - -/*recommended message passing API*/ -void PCU_Comm_Begin(void); -int PCU_Comm_Pack(int to_rank, const void* data, size_t size); -#define PCU_COMM_PACK(to_rank,object)\ -PCU_Comm_Pack(to_rank,&(object),sizeof(object)) -int PCU_Comm_Send(void); -bool PCU_Comm_Receive(void); -bool PCU_Comm_Listen(void); -int PCU_Comm_Sender(void); -bool PCU_Comm_Unpacked(void); -int PCU_Comm_Unpack(void* data, size_t size); -#define PCU_COMM_UNPACK(object)\ -PCU_Comm_Unpack(&(object),sizeof(object)) - -/*turns deterministic ordering for the - above API on/off*/ -void PCU_Comm_Order(bool on); - -/*collective operations*/ -void PCU_Barrier(void); -void PCU_Add_Doubles(double* p, size_t n); -double PCU_Add_Double(double x); -void PCU_Min_Doubles(double* p, size_t n); -double PCU_Min_Double(double x); -void PCU_Max_Doubles(double* p, size_t n); -double PCU_Max_Double(double x); -void PCU_Add_Ints(int* p, size_t n); -int PCU_Add_Int(int x); -void PCU_Add_Longs(long* p, size_t n); -long PCU_Add_Long(long x); -void PCU_Exscan_Ints(int* p, size_t n); -int PCU_Exscan_Int(int x); -void PCU_Exscan_Longs(long* p, size_t n); -long PCU_Exscan_Long(long x); -void PCU_Add_SizeTs(size_t* p, size_t n); -size_t PCU_Add_SizeT(size_t x); -void PCU_Min_SizeTs(size_t* p, size_t n); -size_t PCU_Min_SizeT(size_t x); -void PCU_Max_SizeTs(size_t* p, size_t n); -size_t PCU_Max_SizeT(size_t x); -void PCU_Min_Ints(int* p, size_t n); -int PCU_Min_Int(int x); -void PCU_Max_Ints(int* p, size_t n); -int PCU_Max_Int(int x); -void PCU_Max_Longs(long* p, size_t n); -long PCU_Max_Long(long x); -int PCU_Or(int c); -int PCU_And(int c); - -/*process-level self/peers (mpi wrappers)*/ -int PCU_Proc_Self(void); -int PCU_Proc_Peers(void); - -/*IPComMan replacement API*/ -int PCU_Comm_Write(int to_rank, const void* data, size_t size); -#define PCU_COMM_WRITE(to,data) \ -PCU_Comm_Write(to,&(data),sizeof(data)) -bool PCU_Comm_Read(int* from_rank, void** data, size_t* size); - -/*Debug file I/O API*/ -void PCU_Debug_Open(void); - -void PCU_Debug_Print(const char* format, ...) PCU_FORMAT_ATTRIBUTE(1,2); -/*lesser-used APIs*/ -bool PCU_Comm_Initialized(void); -int PCU_Comm_Packed(int to_rank, size_t* size); -int PCU_Comm_From(int* from_rank); -int PCU_Comm_Received(size_t* size); -void* PCU_Comm_Extract(size_t size); -int PCU_Comm_Rank(int* rank); -int PCU_Comm_Size(int* size); - -/*deprecated method enum*/ -#ifdef __cplusplus -enum PCU_Method { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD }; -#else -typedef enum { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD } PCU_Method; -#endif -int PCU_Comm_Start(PCU_Method method); - -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm(MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm(void); - -/*stack trace helpers using GNU/Linux*/ -void PCU_Protect(void); - -/*MPI_Wtime() equivalent*/ -double PCU_Time(void); - -/*Memory usage*/ -double PCU_GetMem(void); - -/*Access global variable*/ - - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/pcu/PCUObj.cc b/pcu/PCUObj.cc index 0e572d883..9cf86af16 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCUObj.cc @@ -147,7 +147,7 @@ void PCU::DebugOpen() noexcept { append(path, bufsize, "%s", "debug"); if (!msg_->file) - msg_->file = pcu_open_parallel2(GetCHandle(), path, "txt"); + msg_->file = pcu_open_parallel(GetCHandle(), path, "txt"); noto_free(path); } diff --git a/pcu/pcu.cc b/pcu/pcu.cc deleted file mode 100644 index d50937364..000000000 --- a/pcu/pcu.cc +++ /dev/null @@ -1,581 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -/** \file pcu.c - \brief The PCU communication interface */ -/** \page pcu PCU - PCU (the Parallel Control Utility) is a library for parallel computation - based on MPI. - PCU provides three things to users: - 1. A hybrid phased message passing system - 2. Hybrid collective operations - - Phased message passing is similar to Bulk Synchronous Parallel. - All messages are exchanged in a phase, which is a collective operation - involving all threads in the parallel program. - During a phase, the following events happen in sequence: - 1. All threads send non-blocking messages to other threads - 2. All threads receive all messages sent to them during this phase - PCU provides termination detection, which is the ability to detect when all - messages have been received without prior knowledge of which threads - are sending to which. - - The API documentation is here: pcu.c -*/ - -#include "PCU.h" -#include "PCUObj.h" -#include "reel.h" -#include - -static pcu::PCU *global_pcu = nullptr; -namespace pcu { - pcu::PCU* PCU_GetGlobal() { return global_pcu; } -} - -extern "C" { -/** \brief Initializes the PCU library. - \details This function must be called by all MPI processes before - calling any other PCU functions. - MPI_Init or MPI_Init_thread should be called before this function. - */ -int PCU_Comm_Init(void) { - if (global_pcu != nullptr) - reel_fail("nested calls to Comm_Init"); - global_pcu = new pcu::PCU(MPI_COMM_WORLD); - return PCU_SUCCESS; -} - -/** \brief Frees all PCU library structures. - \details This function must be called by all MPI processes after all - other calls to PCU, and before calling MPI_Finalize. - */ -int PCU_Comm_Free(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Free called before Comm_Init"); - delete global_pcu; - global_pcu = nullptr; - return PCU_SUCCESS; -} - -/** \brief Returns the communication rank of the calling thread. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_rank(MPI_COMM_WORLD,rank). - - Ranks are consecutive from 0 to \f$pt-1\f$ for a program with - \f$p\f$ processes and \f$t\f$ threads per process. - Ranks are contiguous within a process, so that the \f$t\f$ threads in process - \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. - */ -int PCU_Comm_Self(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Self called before Comm_Init"); - return global_pcu->Self(); -} - -/** \brief Returns the number of threads in the program. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). - */ -int PCU_Comm_Peers(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Peers called before Comm_Init"); - return global_pcu->Peers(); -} - -/** \brief Begins a PCU communication phase. - \details This function must be called by all threads in the MPI job - at the beginning of each phase of communication. - After calling this function, each thread may call functions like - PCU_Comm_Pack or PCU_Comm_Write. -*/ -void PCU_Comm_Begin(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Begin called before Comm_Init"); - global_pcu->Begin(); -} - -/** \brief Packs data to be sent to \a to_rank. - \details This function appends the block of \a size bytes starting - at \a data to the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Pack(int to_rank, const void *data, size_t size) { - if (global_pcu == nullptr) - reel_fail("Comm_Pack called before Comm_Init"); - return global_pcu->Pack(to_rank, data, size); -} - -/** \brief Sends all buffers for this communication phase. - \details This function should be called by all threads in the MPI job - after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls - to PCU_Comm_Listen or PCU_Comm_Read. - All buffers from this thread are sent out and receiving - may begin after this call. - */ -int PCU_Comm_Send(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Send called before Comm_Init"); - return global_pcu->Send(); -} - -/** \brief Tries to receive a buffer for this communication phase. - \details Either this function or PCU_Comm_Read should be called at least - once by all threads during the communication phase, after PCU_Comm_Send - is called. The result will be false if and only if the communication phase - is over and there are no more buffers to receive. - Otherwise, a buffer was received. - Its contents are retrievable through PCU_Comm_Unpack, and its metadata through - PCU_Comm_Sender and PCU_Comm_Received. - Users should unpack all data from this buffer before calling this function - again, because the previously received buffer is destroyed by the call. - */ -bool PCU_Comm_Listen(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Listen called before Comm_Init"); - return global_pcu->Listen(); -} - -/** \brief Returns in * \a from_rank the sender of the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - */ -int PCU_Comm_Sender(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Sender called before Comm_Init"); - return global_pcu->Sender(); -} - -/** \brief Returns true if the current received buffer has been unpacked. - \details This function should be called after a successful PCU_Comm_Listen. - */ -bool PCU_Comm_Unpacked(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Unpacked called before Comm_Init"); - return global_pcu->Unpacked(); -} - -/** \brief Unpacks a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - \a data must point to a block of memory of at least \a size bytes, into - which the next \a size bytes of the current received buffer will be written. - Subsequent calls will begin unpacking where this call left off, - so that the entire received buffer can be unpacked by a sequence of calls to - this function. - Users must ensure that there remain \a size bytes to be unpacked, - PCU_Comm_Unpacked can help with this. - */ -int PCU_Comm_Unpack(void *data, size_t size) { - if (global_pcu == nullptr) - reel_fail("Comm_Unpack called before Comm_Init"); - return global_pcu->Unpack(data, size); -} - -void PCU_Comm_Order(bool on) { - if (global_pcu == nullptr) - reel_fail("Comm_Order called before Comm_Init"); - global_pcu->Order(on); -} - -/** \brief Blocking barrier over all threads. */ -void PCU_Barrier(void) { - if (global_pcu == nullptr) - reel_fail("Barrier called before Comm_Init"); - global_pcu->Barrier(); -} - -/** \brief Performs an Allreduce sum of double arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n doubles. - After this call, p[i] will contain the sum of all p[i]'s - given by each rank. - */ -void PCU_Add_Doubles(double *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Add_Doubles called before Comm_Init"); - global_pcu->Add(p, n); -} - -double PCU_Add_Double(double x) { - if (global_pcu == nullptr) - reel_fail("Add_Double called before Comm_Init"); - return global_pcu->Add(x); -} - -/** \brief Performs an Allreduce minimum of double arrays. - */ -void PCU_Min_Doubles(double *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Min_Doubles called before Comm_Init"); - global_pcu->Min(p, n); -} - -double PCU_Min_Double(double x) { - if (global_pcu == nullptr) - reel_fail("Min_Double called before Comm_Init"); - return global_pcu->Min(x); -} - -/** \brief Performs an Allreduce maximum of double arrays. - */ -void PCU_Max_Doubles(double *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Max_Doubles called before Comm_Init"); - global_pcu->Max(p, n); -} - -double PCU_Max_Double(double x) { - if (global_pcu == nullptr) - reel_fail("Max_Double called before Comm_Init"); - return global_pcu->Max(x); -} - -/** \brief Performs an Allreduce sum of integers - */ -void PCU_Add_Ints(int *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Add_Ints called before Comm_Init"); - global_pcu->Add(p, n); -} - -int PCU_Add_Int(int x) { - if (global_pcu == nullptr) - reel_fail("Add_Int called before Comm_Init"); - return global_pcu->Add(x); -} - -/** \brief Performs an Allreduce sum of long integers - */ -void PCU_Add_Longs(long *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Add_Longs called before Comm_Init"); - global_pcu->Add(p, n); -} - -long PCU_Add_Long(long x) { - if (global_pcu == nullptr) - reel_fail("Add_Long called before Comm_Init"); - return global_pcu->Add(x); -} - -/** \brief Performs an Allreduce sum of size_t unsigned integers - */ -void PCU_Add_SizeTs(size_t *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Add_SizeTs called before Comm_Init"); - global_pcu->Add(p, n); -} - -size_t PCU_Add_SizeT(size_t x) { - if (global_pcu == nullptr) - reel_fail("Add_SizeT called before Comm_Init"); - return global_pcu->Add(x); -} - -/** \brief Performs an Allreduce minimum of size_t unsigned integers - */ -void PCU_Min_SizeTs(size_t *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Min_SizeTs called before Comm_Init"); - global_pcu->Min(p, n); -} - -size_t PCU_Min_SizeT(size_t x) { - if (global_pcu == nullptr) - reel_fail("Min_SizeT called before Comm_Init"); - return global_pcu->Min(x); -} - -/** \brief Performs an Allreduce maximum of size_t unsigned integers - */ -void PCU_Max_SizeTs(size_t *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Max_SizeTs called before Comm_Init"); - global_pcu->Max(p, n); -} - -size_t PCU_Max_SizeT(size_t x) { - if (global_pcu == nullptr) - reel_fail("Max_SizeT called before Comm_Init"); - return global_pcu->Max(x); -} - -/** \brief Performs an exclusive prefix sum of integer arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n integers. - After this call, p[i] will contain the sum of all p[i]'s - given by ranks lower than the calling rank. - */ -void PCU_Exscan_Ints(int *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Exscan_Ints called before Comm_Init"); - global_pcu->Exscan(p, n); -} - -int PCU_Exscan_Int(int x) { - if (global_pcu == nullptr) - reel_fail("Exscan_Int called before Comm_Init"); - return global_pcu->Exscan(x); -} - -/** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs(long *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Exscan_Longs called before Comm_Init"); - global_pcu->Exscan(p, n); -} - -long PCU_Exscan_Long(long x) { - if (global_pcu == nullptr) - reel_fail("Exscan_Long called before Comm_Init"); - return global_pcu->Exscan(x); -} - -/** \brief Performs an Allreduce minimum of int arrays. - */ -void PCU_Min_Ints(int *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Min_Ints called before Comm_Init"); - global_pcu->Min(p, n); -} - -int PCU_Min_Int(int x) { - if (global_pcu == nullptr) - reel_fail("Min_Int called before Comm_Init"); - return global_pcu->Min(x); -} - -/** \brief Performs an Allreduce maximum of int arrays. - */ -void PCU_Max_Ints(int *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Max_Ints called before Comm_Init"); - global_pcu->Max(p, n); -} - -int PCU_Max_Int(int x) { - if (global_pcu == nullptr) - reel_fail("Max_Int called before Comm_Init"); - return global_pcu->Max(x); -} -/** \brief Performs an Allreduce maximum of long arrays. - */ -void PCU_Max_Longs(long *p, size_t n) { - if (global_pcu == nullptr) - reel_fail("Max_Longs called before Comm_Init"); - global_pcu->Max(p, n); -} - -long PCU_Max_Long(long x) { - if (global_pcu == nullptr) - reel_fail("Max_Long called before Comm_Init"); - return global_pcu->Max(x); -} - -/** \brief Performs a parallel logical OR reduction - */ -int PCU_Or(int c) { - if (global_pcu == nullptr) - reel_fail("Or called before Comm_Init"); - return global_pcu->Or(c); -} - -/** \brief Performs a parallel logical AND reduction - */ -int PCU_And(int c) { - if (global_pcu == nullptr) - reel_fail("And called before Comm_Init"); - return global_pcu->And(c); -} - -/** \brief Returns the unique rank of the calling process. - */ -int PCU_Proc_Self(void) { - if (global_pcu == nullptr) - reel_fail("Proc_Self called before Comm_Init"); - return global_pcu->Self(); -} - -/** \brief Returns the number of processes. - */ -int PCU_Proc_Peers(void) { - if (global_pcu == nullptr) - reel_fail("Proc_Peers called before Comm_Init"); - return global_pcu->Peers(); -} - -/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. - */ -int PCU_Comm_Rank(int *rank) { - if (global_pcu == nullptr) - reel_fail("Comm_Rank called before Comm_Init"); - *rank = global_pcu->Self(); - return PCU_SUCCESS; -} - -/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size(int *size) { - if (global_pcu == nullptr) - reel_fail("Comm_Size called before Comm_Init"); - *size = global_pcu->Peers(); - return PCU_SUCCESS; -} - -/** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized(void) { return global_pcu != nullptr; } - -/** \brief Deprecated, see PCU_Comm_Begin. - */ -int PCU_Comm_Start(PCU_Method method) { - (void)method; // warning silencer - global_pcu->Begin(); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. - \details Returns the size of the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Packed(int to_rank, size_t *size) { - if (global_pcu == nullptr) - reel_fail("Comm_Packed called before Comm_Init"); - return global_pcu->Packed(to_rank, size); -} - -/** \brief Packs a message to be sent to \a to_rank. - \details This function packs a message into the buffer being sent - to \a to_rank. - Messages packed by this function can be received using the function - PCU_Comm_Read. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - If this function is used, PCU_Comm_Pack should not be used. - */ -int PCU_Comm_Write(int to_rank, const void *data, size_t size) { - if (global_pcu == nullptr) - reel_fail("Comm_Write called before Comm_Init"); - return global_pcu->Write(to_rank, data, size); -} - -/** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive(void) { - if (global_pcu == nullptr) - reel_fail("Comm_Receive called before Comm_Init"); - return global_pcu->Receive(); -} - -/** \brief Receives a message for this communication phase. - \details This function tries to receive a message packed by - PCU_Comm_Write. - If a the communication phase is over and there are no more - messages to receive, this function returns false. - Otherwise, * \a from_rank will be the rank which sent the message, - * \a data will point to the start of the message data, and - * \a size will be the number of bytes of message data. - If this function is used, PCU_Comm_Receive should not be used. - Note that the address * \a data points into a PCU buffer, so - it is strongly recommended that this data be read and not modified. - */ -bool PCU_Comm_Read(int *from_rank, void **data, size_t *size) { - if (global_pcu == nullptr) - reel_fail("Comm_Read called before Comm_Init"); - return global_pcu->Read(from_rank, data, size); -} - -void PCU_Debug_Open(void) { - if (global_pcu == nullptr) - reel_fail("Debug_Open called before Comm_Init"); - global_pcu->DebugOpen(); -} - -/** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print(const char *format, ...) { - if (global_pcu == nullptr) - reel_fail("Debug_Print called before Comm_Init"); - va_list arglist; - va_start(arglist, format); - global_pcu->DebugPrint(format, arglist); - va_end(arglist); -} - -/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From(int *from_rank) { - if (global_pcu == nullptr) - reel_fail("Comm_From called before Comm_Init"); - return global_pcu->From(from_rank); -} - -/** \brief Returns in * \a size the bytes in the current received buffer - \details This function should be called after a successful PCU_Comm_Receive. - The size returned will be the total received size regardless of how - much unpacking has been done. - */ -int PCU_Comm_Received(size_t *size) { - if (global_pcu == nullptr) - reel_fail("Comm_Received called before Comm_Init"); - return global_pcu->Received(size); -} - -/** \brief Extracts a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Receive. - The next \a size bytes of the current received buffer are unpacked, - and an internal pointer to that data is returned. - The returned pointer must not be freed by the user. - */ -void *PCU_Comm_Extract(size_t size) { - if (global_pcu == nullptr) - reel_fail("Comm_Extract called before Comm_Init"); - return global_pcu->Extract(size); -} - -/** \brief Reinitializes PCU with a new MPI communicator. - \details All of PCU's logic is based off two duplicates - of this communicator, so you can safely get PCU to act - on sub-groups of processes using this function. - This call should be collective over all processes - in the previous communicator. This is a very heavy weight function - and should be used sparingly. - */ -void PCU_Switch_Comm(MPI_Comm new_comm) { - if (global_pcu == nullptr) - reel_fail("Switch_Comm called before Comm_Init"); - global_pcu->SwitchMPIComm(new_comm); -} - -/** \brief Return the current MPI communicator - \details Returns the communicator given to the - most recent PCU_Switch_Comm call, or MPI_COMM_WORLD - otherwise. - */ -MPI_Comm PCU_Get_Comm(void) { - if (global_pcu == nullptr) - reel_fail("Get_Comm called before Comm_Init"); - return global_pcu->GetMPIComm(); -} - -/** \brief Return the time in seconds since some time in the past - */ -double PCU_Time(void) { return pcu::Time(); } - -void PCU_Protect(void) { return pcu::Protect(); } - -double PCU_GetMem(void) { return pcu::GetMem(); } - - -/** \brief Return the global PCU - */ - -PCUHandle PCU_Get_Global_Handle(void) { - PCUHandle h; - h.ptr = static_cast(pcu::PCU_GetGlobal()); - return h; -} - -} diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index b22ab51fc..93ad2345c 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -158,12 +158,7 @@ static void close_compressed(pcu_file* pf) * Ideally, the filesystem handles the load and this can be * removed. */ -FILE* pcu_group_open(const char* path, bool write) { - PCUHandle h = PCU_Get_Global_Handle(); - return pcu_group_open2(h, path, write); -} - -FILE* pcu_group_open2(PCUHandle h, const char* path, bool write) { +FILE* pcu_group_open(PCUHandle h, const char* path, bool write) { FILE* fp = NULL; const int rank = PCU_Comm_Self2(h); const char* mode = write ? "w" : "r"; @@ -187,18 +182,12 @@ FILE* pcu_group_open2(PCUHandle h, const char* path, bool write) { return fp; } -pcu_file* pcu_fopen(const char* name, bool write, bool compress) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return pcu_fopen2(h, name, write, compress); -} - -pcu_file* pcu_fopen2(PCUHandle h, const char* name, bool write, bool compress) +pcu_file* pcu_fopen(PCUHandle h, const char* name, bool write, bool compress) { pcu_file* pf = (pcu_file*) malloc(sizeof(pcu_file)); pf->compress = compress; pf->write = write; - pf->f = pcu_group_open2(h, name, write); + pf->f = pcu_group_open(h, name, write); if (!pf->f) { perror("pcu_fopen"); reel_fail("pcu_fopen couldn't open \"%s\"", name); @@ -366,13 +355,7 @@ void pcu_write_string (pcu_file * f, const char * p) pcu_write (f, p, len + 1); } -FILE* pcu_open_parallel(const char* prefix, const char* ext) -{ - PCUHandle h = PCU_Get_Global_Handle(); - return pcu_open_parallel2(h, prefix, ext); -} - -FILE* pcu_open_parallel2(PCUHandle h, const char* prefix, const char* ext) +FILE* pcu_open_parallel(PCUHandle h, const char* prefix, const char* ext) { //max_rank_chars = strlen("4294967296"), 4294967296 = 2^32 ~= INT_MAX static const size_t max_rank_chars = 10; diff --git a/pcu/pcu_io.h b/pcu/pcu_io.h index ef7eaad0c..88edece46 100644 --- a/pcu/pcu_io.h +++ b/pcu/pcu_io.h @@ -22,8 +22,7 @@ extern "C" { struct pcu_file; typedef struct PCUHandle PCUHandle; -struct pcu_file* pcu_fopen(const char* path, bool write, bool compress); -struct pcu_file* pcu_fopen2(PCUHandle h, const char* path, bool write, bool compress); +struct pcu_file* pcu_fopen(PCUHandle h, const char* path, bool write, bool compress); void pcu_fclose (struct pcu_file * pf); void pcu_read(struct pcu_file* f, char* p, size_t n); void pcu_write(struct pcu_file* f, const char* p, size_t n); @@ -36,10 +35,8 @@ void pcu_write_doubles(struct pcu_file* f, double* p, size_t n); void pcu_read_string(struct pcu_file* f, char** p); void pcu_write_string(struct pcu_file* f, const char* p); -FILE* pcu_open_parallel(const char* prefix, const char* ext); -FILE* pcu_open_parallel2(PCUHandle h, const char* prefix, const char* ext); -FILE* pcu_group_open(const char* path, bool write); -FILE* pcu_group_open2(PCUHandle h, const char* path, bool write); +FILE* pcu_open_parallel(PCUHandle h, const char* prefix, const char* ext); +FILE* pcu_group_open(PCUHandle h, const char* path, bool write); void pcu_swap_doubles(double* p, size_t n); void pcu_swap_unsigneds(unsigned* p, size_t n); diff --git a/phasta/ph.h b/phasta/ph.h index e5b8b9dc9..0b869bb1e 100644 --- a/phasta/ph.h +++ b/phasta/ph.h @@ -16,7 +16,7 @@ void setupInputSubdir(std::string& path, pcu::PCU *PCUObj); void setupOutputSubdir(std::string& path, pcu::PCU *PCUObj, bool all_mkdir=false); void writeAuxiliaryFiles(std::string path, int timestep_or_dat, pcu::PCU *PCUObj); bool mesh_has_ext(const char* filename, const char* ext); -apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj = nullptr); +apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj); } diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 999246376..df973f6dc 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -103,15 +103,15 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, }//end namespace namespace chef { - static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU *PCUObj) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open(path, false);) + PHASTAIO_OPENTIME(f = pcu_group_open(PCUObj->GetCHandle(), path, false);) return f; } - static FILE* openfile_write(ph::Output&, const char* path) { + static FILE* openfile_write(ph::Output& out, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open(path, true);) + PHASTAIO_OPENTIME(f = pcu_group_open(out.mesh->getPCU()->GetCHandle(), path, true);) return f; } diff --git a/phasta/phIO.c b/phasta/phIO.c index d0517b6bc..50b5a4845 100644 --- a/phasta/phIO.c +++ b/phasta/phIO.c @@ -162,18 +162,16 @@ static void parse_params(char* header, long* bytes, *step = params[STEP_PARAM]; } -int ph_should_swap(FILE* f) { - PCUHandle h = PCU_Get_Global_Handle(); +int ph_should_swap(FILE* f, PCUHandle h) { return read_magic_number(h, f); } -int ph_read_field(FILE* f, const char* field, int swap, +int ph_read_field(FILE* f, PCUHandle h, const char* field, int swap, double** data, int* nodes, int* vars, int* step, char* hname) { long bytes, n; char header[PH_LINE]; int ok; - PCUHandle h = PCU_Get_Global_Handle(); ok = find_header(h, f, field, hname, header); if(!ok) /* not found */ return 0; diff --git a/phasta/phIO.h b/phasta/phIO.h index 4c9bbebbe..921a2c0c1 100644 --- a/phasta/phIO.h +++ b/phasta/phIO.h @@ -18,14 +18,14 @@ void ph_write_ints(FILE* f, const char* name, int* data, * swapped to account for endianness * @return 1 if swapping is required, 0 otherwise */ -int ph_should_swap(FILE* f); +int ph_should_swap(FILE* f, PCUHandle h); /** * @brief read a field * @return 1 if no data block was read, 2 if data block read, 0 otherwise */ -int ph_read_field(FILE* f, const char* field, int swap, +int ph_read_field(FILE* f, PCUHandle h, const char* field, int swap, double** data, int* nodes, int* vars, int* step, char* hname); void ph_write_field(FILE* f, const char* field, double* data, int nodes, int vars, int step); diff --git a/phasta/phRestart.cc b/phasta/phRestart.cc index e403df471..3410a8b47 100644 --- a/phasta/phRestart.cc +++ b/phasta/phRestart.cc @@ -343,8 +343,8 @@ int readAndAttachField( int nodes, vars, step; char hname[1024]; const char* anyfield = ""; - int ret = ph_read_field(f, anyfield, swap, - &data, &nodes, &vars, &step, hname); + int ret = ph_read_field(f, m->getPCU()->GetCHandle(), + anyfield, swap, &data, &nodes, &vars, &step, hname); /* no field was found or the field has an empty data block */ if(ret==0 || ret==1) return ret; @@ -478,7 +478,7 @@ void readAndAttachFields(Input& in, apf::Mesh* m) { lion_eprint(1,"failed to open \"%s\"!\n", filename.c_str()); abort(); } - int swap = ph_should_swap(f); + int swap = ph_should_swap(f, m->getPCU()->GetCHandle()); /* stops when ph_read_field returns 0 */ while( readAndAttachField(in,f,m,swap) ) {} PHASTAIO_CLOSETIME(fclose(f);) diff --git a/pumi/pumi.h b/pumi/pumi.h index f9a1c628f..bfdf4e346 100644 --- a/pumi/pumi.h +++ b/pumi/pumi.h @@ -197,7 +197,7 @@ void pumi_gent_getEntArrTag (pGeomEnt ent, pTag tag, pGeomEnt** data, int* data_ //************************************ // create an empty mesh -pMesh pumi_mesh_create(pGeom g, int mesh_dim, bool periodic=false); +pMesh pumi_mesh_create(pGeom g, int mesh_dim, pcu::PCU *PCUObj, bool periodic=false); void pumi_mesh_freeze(pMesh m); pMeshEnt pumi_mesh_createVtx(pMesh m, pGeomEnt ge, double* xyz); //ent_topology: VERTEX (0), EDGE (1), TRIANGLE (2), QUAD (3), TET (4), HEX (5), PRISM (6), PYRAMID (7) diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 5b6540da9..735e48ff2 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -28,9 +28,9 @@ using std::map; // mesh creation -pMesh pumi_mesh_create(pGeom g, int mesh_dim, bool periodic) +pMesh pumi_mesh_create(pGeom g, int mesh_dim, pcu::PCU *PCUObj, bool periodic) { - pumi::instance()->mesh = apf::makeEmptyMdsMesh(g->getGmi(), mesh_dim, periodic); + pumi::instance()->mesh = apf::makeEmptyMdsMesh(g->getGmi(), mesh_dim, periodic, PCUObj); return pumi::instance()->mesh; } @@ -203,27 +203,6 @@ pGeom pumi_mesh_getGeom(pMesh) return pumi::instance()->model; } -// load a serial mesh on master process then distribute as per the distribution object -/* -pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, const char* mesh_type) -{ - if (strcmp(mesh_type,"mds")) - { - if (!PCU_Comm_Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getGmi(), filename); - merge_comm(prevComm); - pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1, m->getPCU()); - return pumi::instance()->mesh; -} -*/ pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, pcu::PCU *PCUObj, const char* mesh_type) diff --git a/sam/samElementCount.cc b/sam/samElementCount.cc index 43368c4f6..10a6bf71e 100644 --- a/sam/samElementCount.cc +++ b/sam/samElementCount.cc @@ -2,7 +2,6 @@ #include #include -#include namespace sam { diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index b24a1b4e0..de9226922 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -38,8 +38,8 @@ static bool overwriteAPFCoord(apf::Mesh2* m) { return true; } -static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { - return pcu_group_open(path, false); +static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU *PCUObj) { + return pcu_group_open(PCUObj->GetCHandle(), path, false); } int main(int argc, char** argv) diff --git a/test/pumi.cc b/test/pumi.cc index 1200d862f..9e13306df 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -638,7 +638,7 @@ void TEST_NEW_MESH(pMesh m) // create an empty mesh pGeom new_g = pumi_geom_load("", m->getPCU(), "null"); - pMesh new_m = pumi_mesh_create(new_g, 2); + pMesh new_m = pumi_mesh_create(new_g, 2, m->getPCU()); double xyz[3]; pMeshIter it = m->begin(1); diff --git a/test/ugrid.cc b/test/ugrid.cc index 17bf7b991..25f1cd00a 100644 --- a/test/ugrid.cc +++ b/test/ugrid.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include diff --git a/test/ugridptnstats.cc b/test/ugridptnstats.cc index 0fffb75b9..9c79bbb44 100644 --- a/test/ugridptnstats.cc +++ b/test/ugridptnstats.cc @@ -29,7 +29,7 @@ int main(int argc, char** argv) const char* ptnfile = argv[2]; gmi_register_null(); gmi_model* g = gmi_load(".null"); - apf::printUgridPtnStats(g,ugridfile,ptnfile,weights); + apf::printUgridPtnStats(g,ugridfile,ptnfile,weights,PCUObj.get()); } MPI_Finalize(); } From 0dd10d119d7c29e52db840a266c269350815f727 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 1 Jul 2024 09:34:27 -0400 Subject: [PATCH 117/141] small fixes --- apf/apfCavityOp.cc | 3 +-- ma/ma.cc | 4 ++-- mds/apfMDS.cc | 3 +-- mds/mdsANSYS.cc | 3 +-- mds/mdsGmsh.cc | 6 ++--- mds/mdsUgrid.cc | 6 ++--- omega_h/apfOmega_h.cc | 2 +- .../maximalIndependentSet/test/testMIS.cc | 2 -- pcu/noto/noto_malloc.h | 8 +++---- phasta/adaptLvlSet_loop.cc | 22 +++++++++---------- phasta/chefStream.cc | 18 +++++++-------- phasta/cut_interface.cc | 6 ++--- phasta/phBC.h | 2 +- phasta/phiotimer.cc | 1 - phasta/threshold.cc | 4 ++-- spr/sprGetGradIPField.cc | 1 - test/ptnParma.cc | 2 +- zoltan/apfZoltan.cc | 4 ++-- 18 files changed, 43 insertions(+), 54 deletions(-) diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 51f2e01a4..4d9097116 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -133,7 +133,7 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = mesh->getPCU()->Min((int)requests.empty()); + int done = mesh->getPCU()->Min(static_cast(requests.empty())); if (done) return false; /* throw in the local pull requests */ int self = mesh->getPCU()->Self(); @@ -166,7 +166,6 @@ bool CavityOp::sendPullRequests(std::vector& received) while ( ! mesh->getPCU()->Unpacked()) { mesh->getPCU()->Unpack(request.e); - //mesh->getPCU()->Unpack(&(request.e), sizeof(request.e)); received.push_back(request); } } diff --git a/ma/ma.cc b/ma/ma.cc index 72d137c67..24c5100ce 100644 --- a/ma/ma.cc +++ b/ma/ma.cc @@ -23,10 +23,10 @@ namespace ma { void adapt(Input* in) { double t0 = pcu::Time(); + print(in->mesh->getPCU(), "version 2.0 !"); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); - print(a->mesh->getPCU(), "version 2.0 !"); for (int i = 0; i < in->maximumIterations; ++i) { print(a->mesh->getPCU(), "iteration %d", i); @@ -63,10 +63,10 @@ void adapt(const Input* in) void adaptVerbose(Input* in, bool verbose) { double t0 = pcu::Time(); + print(in->mesh->getPCU(), "version 2.0 - dev !"); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); - print(a->mesh->getPCU(), "version 2.0 - dev !"); for (int i = 0; i < in->maximumIterations; ++i) { print(a->mesh->getPCU(), "iteration %d", i); diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 77fe67180..09ce7714d 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -780,8 +780,7 @@ class MeshMDS : public Mesh2 Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj) { - Mesh2* m; - m = new MeshMDS(model, dim, isMatched, PCUObj); + Mesh2* m = new MeshMDS(model, dim, isMatched, PCUObj); initResidence(m, dim); return m; } diff --git a/mds/mdsANSYS.cc b/mds/mdsANSYS.cc index 9ee145f76..3bd0a8eb6 100644 --- a/mds/mdsANSYS.cc +++ b/mds/mdsANSYS.cc @@ -156,8 +156,7 @@ Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PC { Nodes nodes; parseNodes(nodefile, nodes); - Mesh2* m; - m = parseElems(elemfile, nodes, PCUObj); + Mesh2* m = parseElems(elemfile, nodes, PCUObj); m->acceptChanges(); deriveMdsModel(m); return m; diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc index 2406467eb..b6d3f8524 100644 --- a/mds/mdsGmsh.cc +++ b/mds/mdsGmsh.cc @@ -461,8 +461,7 @@ void gmshFindDmg(const char* fnameDmg, const char* filename) Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m; - m = makeEmptyMdsMesh(g, 0, false, PCUObj); + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); readGmsh(m, filename); return m; } @@ -470,8 +469,7 @@ Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename, pcu::PCU *PCUObj) { gmshFindDmg(fnameDmg, filename); // new function that scans $Entities and writes a dmg - Mesh2* m; - m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); + Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); readGmsh(m, filename); return m; } diff --git a/mds/mdsUgrid.cc b/mds/mdsUgrid.cc index df4dc5ac4..a96afabe5 100644 --- a/mds/mdsUgrid.cc +++ b/mds/mdsUgrid.cc @@ -663,8 +663,7 @@ namespace { namespace apf { Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m; - m = makeEmptyMdsMesh(g, 0, false, PCUObj); + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); header hdr; Reader r; initReader(&r, m, filename); @@ -686,8 +685,7 @@ namespace apf { void printUgridPtnStats(gmi_model* g, const char* ufile, const char* vtxptn, const double elmWeights[], pcu::PCU *PCUObj) { - Mesh2* m; - m = makeEmptyMdsMesh(g, 0, false, PCUObj); + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); apf::changeMdsDimension(m, 3); printPtnStats(m, ufile, vtxptn, elmWeights); m->destroyNative(); diff --git a/omega_h/apfOmega_h.cc b/omega_h/apfOmega_h.cc index d6134c175..663831af8 100644 --- a/omega_h/apfOmega_h.cc +++ b/omega_h/apfOmega_h.cc @@ -388,7 +388,7 @@ static void owners_from_osh( for (int i = 0; i < om->nents(ent_dim); ++i) { osh::LO tmp = own_ids[i]; am->getPCU()->Pack(own_ranks[i], tmp); - am->getPCU()->(own_ranks[i], ents[i]); + am->getPCU()->Pack(own_ranks[i], ents[i]); } am->getPCU()->Send(); while (am->getPCU()->Receive()) { diff --git a/parma/diffMC/maximalIndependentSet/test/testMIS.cc b/parma/diffMC/maximalIndependentSet/test/testMIS.cc index 68882d35d..38429b5c2 100644 --- a/parma/diffMC/maximalIndependentSet/test/testMIS.cc +++ b/parma/diffMC/maximalIndependentSet/test/testMIS.cc @@ -715,9 +715,7 @@ int main(int argc, char** argv) { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); pcu::Protect(); int rank = PCUObj.get()->Self(); - //PCU_Comm_Rank(&rank); int commSize = PCUObj.get()->Peers(); - //PCU_Comm_Size(&commSize); int opt; int debugMode = 0; diff --git a/pcu/noto/noto_malloc.h b/pcu/noto/noto_malloc.h index 74a2c4bac..42a0cd24a 100644 --- a/pcu/noto/noto_malloc.h +++ b/pcu/noto/noto_malloc.h @@ -16,10 +16,10 @@ extern "C" { #include -void *noto_malloc(size_t size); -#define NOTO_MALLOC(p, c) ((p) = noto_malloc(sizeof(*(p)) * (c))) -void *noto_realloc(void *p, size_t size); -void noto_free(void *p); +void* noto_malloc(size_t size); +#define NOTO_MALLOC(p,c) ((p)=noto_malloc(sizeof(*(p))*(c))) +void* noto_realloc(void* p, size_t size); +void noto_free(void* p); size_t noto_malloced(void); #ifdef __cplusplus } diff --git a/phasta/adaptLvlSet_loop.cc b/phasta/adaptLvlSet_loop.cc index 08832a371..042c46b46 100644 --- a/phasta/adaptLvlSet_loop.cc +++ b/phasta/adaptLvlSet_loop.cc @@ -39,7 +39,7 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); @@ -49,23 +49,23 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(expanded_pcu_obj.get()); + GRStream* grs = makeGRStream(PCUObj.get()); ph::Input ctrl; - ctrl.load("adaptLvlSet.inp", expanded_pcu_obj.get()); + ctrl.load("adaptLvlSet.inp", PCUObj.get()); //preprocess (define bubbles) - chef::cook(g,m,ctrl,grs,expanded_pcu_obj.get()); - RStream* rs = makeRStream(expanded_pcu_obj.get()); - attachRStream(grs,rs,expanded_pcu_obj.get()); + chef::cook(g,m,ctrl,grs,PCUObj.get()); + RStream* rs = makeRStream(PCUObj.get()); + attachRStream(grs,rs,PCUObj.get()); reconfigureChef(ctrl); // attach the solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs,grs,expanded_pcu_obj.get()); - attachRStream(grs,rs,expanded_pcu_obj.get()); + chef::cook(g,m,ctrl,rs,grs,PCUObj.get()); + attachRStream(grs,rs,PCUObj.get()); // attach a solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs,expanded_pcu_obj.get()); - destroyGRStream(grs,expanded_pcu_obj.get()); - destroyRStream(rs,expanded_pcu_obj.get()); + chef::cook(g,m,ctrl,rs,PCUObj.get()); + destroyGRStream(grs,PCUObj.get()); + destroyRStream(rs,PCUObj.get()); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/phasta/chefStream.cc b/phasta/chefStream.cc index 11f6d2d08..3f4258184 100644 --- a/phasta/chefStream.cc +++ b/phasta/chefStream.cc @@ -35,7 +35,7 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); @@ -45,16 +45,16 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(expanded_pcu_obj.get()); + GRStream* grs = makeGRStream(PCUObj.get()); ph::Input ctrl; - ctrl.load("adapt.inp", expanded_pcu_obj.get()); - chef::cook(g,m,ctrl,grs,expanded_pcu_obj.get()); - RStream* rs = makeRStream(expanded_pcu_obj.get()); - attachRStream(grs,rs,expanded_pcu_obj.get()); + ctrl.load("adapt.inp", PCUObj.get()); + chef::cook(g,m,ctrl,grs,PCUObj.get()); + RStream* rs = makeRStream(PCUObj.get()); + attachRStream(grs,rs,PCUObj.get()); reconfigureChef(ctrl); - chef::cook(g,m,ctrl,rs,expanded_pcu_obj.get()); - destroyGRStream(grs,expanded_pcu_obj.get()); - destroyRStream(rs,expanded_pcu_obj.get()); + chef::cook(g,m,ctrl,rs,PCUObj.get()); + destroyGRStream(grs,PCUObj.get()); + destroyRStream(rs,PCUObj.get()); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/phasta/cut_interface.cc b/phasta/cut_interface.cc index 09e30d713..151ae0885 100644 --- a/phasta/cut_interface.cc +++ b/phasta/cut_interface.cc @@ -33,8 +33,8 @@ int main(int argc, char** argv) return 0; } { - auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - PCU_ALWAYS_ASSERT(expanded_pcu_obj.get()->Peers() == 1); + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 1); #ifdef HAVE_SIMMETRIX SimModel_start(); Sim_readLicenseFile(0); @@ -61,7 +61,7 @@ int main(int argc, char** argv) gm = gmi_sim_load(modelfile, attribfile); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); - apf::Mesh2* m = ph::loadMesh(gm, meshfile, expanded_pcu_obj.get()); + apf::Mesh2* m = ph::loadMesh(gm, meshfile, PCUObj.get()); m->verify(); #ifdef HAVE_SIMMETRIX if (ph::mesh_has_ext(meshfile, "sms")) diff --git a/phasta/phBC.h b/phasta/phBC.h index 761de8d1c..fe88fdf1b 100644 --- a/phasta/phBC.h +++ b/phasta/phBC.h @@ -7,7 +7,7 @@ #include #include #include "phInput.h" -//#include + /* full names and abbreviations for boundary conditions: Essential boundary conditions: diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index 4444b1eee..c5af90606 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -281,7 +281,6 @@ void phastaio_printStats(PCUHandle h) { } void phastaio_initStats(PCUHandle h) { - //removed checking if PCU is initiallized as it's a class invarient of apf::Mesh that it is (void) h; #ifdef __INTEL_COMPILER phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(h); diff --git a/phasta/threshold.cc b/phasta/threshold.cc index 905a13cea..f8e807d80 100644 --- a/phasta/threshold.cc +++ b/phasta/threshold.cc @@ -21,7 +21,7 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -30,7 +30,7 @@ int main(int argc, char** argv) gmi_register_mesh(); lion_set_verbosity(1); ph::Input in; - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], expanded_pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); m->verify(); in.restartFileName = argv[3]; in.timeStepNumber = 0; diff --git a/spr/sprGetGradIPField.cc b/spr/sprGetGradIPField.cc index af4e49935..94be3d4d9 100644 --- a/spr/sprGetGradIPField.cc +++ b/spr/sprGetGradIPField.cc @@ -10,7 +10,6 @@ #include "apfShape.h" #include - namespace spr { apf::Field* getGradIPField(apf::Field* f, const char* name, int order) diff --git a/test/ptnParma.cc b/test/ptnParma.cc index 71eff01f3..d1e767746 100644 --- a/test/ptnParma.cc +++ b/test/ptnParma.cc @@ -121,7 +121,7 @@ void mymain(bool ismaster, pcu::PCU *PCUObj) gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = NULL; apf::Migration* plan = NULL; - pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj) + pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj); if (ismaster) { m = apf::loadMdsMesh(modelFile,meshFile,groupedPCUObj); Parma_PrintPtnStats(m, "initial"); diff --git a/zoltan/apfZoltan.cc b/zoltan/apfZoltan.cc index b90866a07..8023211bb 100644 --- a/zoltan/apfZoltan.cc +++ b/zoltan/apfZoltan.cc @@ -30,12 +30,12 @@ class ZoltanSplitter : public Splitter for (int i = 0; i < plan->count(); ++i) { MeshEntity* e = plan->get(i); int p = plan->sending(e); - p += bridge->mesh->getPCU()->Self() * multiple; + p += bridge.mesh->getPCU()->Self() * multiple; plan->send(e, p); } } double t1 = pcu::Time(); - if (!bridge->mesh->getPCU()->Self()) + if (!bridge.mesh->getPCU()->Self()) lion_oprint(1, "planned Zoltan split factor %d to target" " imbalance %f in %f seconds\n", multiple, tolerance, t1 - t0); return plan; From 6e9a87525abfbd247bc2844f5cb79a3e7a64b886 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 5 Jul 2024 12:27:00 -0400 Subject: [PATCH 118/141] PCU2 using PCU naming, changed PCUHandle to PCU_t --- mds/apfMDS.cc | 1 - mds/apfMDS.h | 6 +- mds/mdsCGNS.cc | 76 +++++++++++------------ mds/mds_apf.c | 38 ++++++------ mds/mds_apf.h | 14 ++--- mds/mds_net.c | 88 +++++++++++++-------------- mds/mds_net.h | 12 ++-- mds/mds_order.c | 26 ++++---- mds/mds_smb.c | 50 ++++++++-------- pcu/CMakeLists.txt | 7 +-- pcu/PCU.h | 120 +++++++++++++++++++++++++++++++++++++ pcu/PCU2.h | 121 ------------------------------------- pcu/PCUObj.h | 2 +- pcu/{pcu2.cc => pcu.cc} | 129 +++++++++++++++++++--------------------- pcu/pcu_defines.h | 2 +- pcu/pcu_io.c | 20 +++---- pcu/pcu_io.h | 8 +-- phasta/phCook.cc | 2 +- phasta/phIO.c | 16 ++--- phasta/phIO.h | 4 +- phasta/phRestart.cc | 2 +- phasta/phiotimer.cc | 46 +++++++------- phasta/phiotimer.h | 7 +-- test/cgns.cc | 4 +- 24 files changed, 395 insertions(+), 406 deletions(-) create mode 100644 pcu/PCU.h delete mode 100644 pcu/PCU2.h rename pcu/{pcu2.cc => pcu.cc} (82%) diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 09ce7714d..7ec95285e 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -8,7 +8,6 @@ *******************************************************************************/ -#include #include #include #include "apfMDS.h" diff --git a/mds/apfMDS.h b/mds/apfMDS.h index 7a5aa1477..a44a2f74d 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -54,7 +54,7 @@ class Field; /** \brief a map from global ids to vertex objects */ typedef std::map GlobalToVert; -typedef struct PCUHandle PCUHandle; +typedef struct PCU_t PCU_t; /** \brief create an empty MDS part @@ -205,11 +205,11 @@ int getMdsIndex(Mesh2* in, MeshEntity* e); MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index); Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); -Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); +Mesh2* loadMdsFromCGNS2(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); // names of mesh data to read from file: (VERTEX, VelocityX; CellCentre, Pressure) Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); -Mesh2* loadMdsFromCGNS2(PCUHandle h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); +Mesh2* loadMdsFromCGNS2(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); int gmshMajorVersion(const char* filename); diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index cd41abb5e..fa2d7e883 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -10,7 +10,7 @@ #include "apfShape.h" #include "gmi.h" /* this is for gmi_getline... */ #include -#include +#include #include #include #include "apfFieldData.h" @@ -200,14 +200,14 @@ struct MeshDataGroup }; template -void DebugParallelPrinter(PCUHandle h, std::ostream &out, Arg &&arg, Args &&... args) +void DebugParallelPrinter(PCU_t h, std::ostream &out, Arg &&arg, Args &&... args) { // if constexpr (debugOutput) // probably will not get away with c++17 if (debugOutput) { - for (int i = 0; i < PCU_Comm_Peers2(h); i++) + for (int i = 0; i < PCU_Comm_Peers(h); i++) { - if (i == PCU_Comm_Self2(h)) + if (i == PCU_Comm_Self(h)) { out << "Rank [" << i << "] " << std::forward(arg); //((out << ", " << std::forward(args)), ...); // probably will not get away with c++17 @@ -223,15 +223,15 @@ void DebugParallelPrinter(PCUHandle h, std::ostream &out, Arg &&arg, Args &&... } template -void Kill(PCUHandle h, const int fid, Args &&... args) +void Kill(PCU_t h, const int fid, Args &&... args) { DebugParallelPrinter(h, std::cout, "***** CGNS ERROR", args...); - if (PCU_Comm_Initialized2(h)) + if (PCU_Comm_Initialized(h)) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free2(h); + PCU_Comm_Free(h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -244,15 +244,15 @@ void Kill(PCUHandle h, const int fid, Args &&... args) } } -void Kill(PCUHandle h, const int fid) +void Kill(PCU_t h, const int fid) { DebugParallelPrinter(h, std::cout, "***** CGNS ERROR"); - if (PCU_Comm_Initialized2(h)) + if (PCU_Comm_Initialized(h)) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free2(h); + PCU_Comm_Free(h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -265,7 +265,7 @@ void Kill(PCUHandle h, const int fid) } } -auto ReadCGNSCoords(PCUHandle h, int cgid, int base, int zone, int ncoords, int nverts, const std::vector &, const apf::GlobalToVert &globalToVert) +auto ReadCGNSCoords(PCU_t h, int cgid, int base, int zone, int ncoords, int nverts, const std::vector &, const apf::GlobalToVert &globalToVert) { // Read min required as defined by consecutive range // make one based as ReadElements makes zero based @@ -332,14 +332,14 @@ auto ReadCGNSCoords(PCUHandle h, int cgid, int base, int zone, int ncoords, int return points; } -void SimpleElementPartition(PCUHandle h, std::vector &numberToReadPerProc, std::vector &startingIndex, int el_start /* one based */, int el_end, int numElements) +void SimpleElementPartition(PCU_t h, std::vector &numberToReadPerProc, std::vector &startingIndex, int el_start /* one based */, int el_end, int numElements) { - numberToReadPerProc.resize(PCU_Comm_Peers2(h), 0); + numberToReadPerProc.resize(PCU_Comm_Peers(h), 0); const cgsize_t blockSize = static_cast( std::floor(static_cast(numElements) / - static_cast(PCU_Comm_Peers2(h)))); + static_cast(PCU_Comm_Peers(h)))); - DebugParallelPrinter(h, std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers2(h)); + DebugParallelPrinter(h, std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers(h)); cgsize_t counter = 0; if (blockSize == 0 && numElements > 0) @@ -349,7 +349,7 @@ void SimpleElementPartition(PCUHandle h, std::vector &numberToReadPerP bool keepGoing = true; while (keepGoing) { - for (int p = 0; p < PCU_Comm_Peers2(h) && keepGoing; p++) + for (int p = 0; p < PCU_Comm_Peers(h) && keepGoing; p++) { if (counter + blockSize <= numElements) { @@ -368,9 +368,9 @@ void SimpleElementPartition(PCUHandle h, std::vector &numberToReadPerP } DebugParallelPrinter(h, std::cout, "Sanity check: Counter", counter, "numElements", numElements); - DebugParallelPrinter(h, std::cout, "numberToReadPerProc for rank", PCU_Comm_Self2(h), "is:", numberToReadPerProc[PCU_Comm_Self2(h)]); + DebugParallelPrinter(h, std::cout, "numberToReadPerProc for rank", PCU_Comm_Self(h), "is:", numberToReadPerProc[PCU_Comm_Self(h)]); - startingIndex.resize(PCU_Comm_Peers2(h), 0); + startingIndex.resize(PCU_Comm_Peers(h), 0); startingIndex[0] = el_start; for (std::size_t i = 1; i < numberToReadPerProc.size(); i++) { @@ -381,7 +381,7 @@ void SimpleElementPartition(PCUHandle h, std::vector &numberToReadPerP DebugParallelPrinter(h, std::cout, "Element start, end, numElements ", el_start, el_end, numElements); - DebugParallelPrinter(h, std::cout, "startingIndex for rank", PCU_Comm_Self2(h), "is:", startingIndex[PCU_Comm_Self2(h)]); + DebugParallelPrinter(h, std::cout, "startingIndex for rank", PCU_Comm_Self(h), "is:", startingIndex[PCU_Comm_Self(h)]); DebugParallelPrinter(h, std::cout, "Returning from SimpleElementPartition \n"); } @@ -389,25 +389,25 @@ void SimpleElementPartition(PCUHandle h, std::vector &numberToReadPerP using Pair = std::pair; using LocalElementRanges = std::vector; // one based -auto ReadElements(PCUHandle h, int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges) +auto ReadElements(PCU_t h, int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges) { std::vector numberToReadPerProc; std::vector startingIndex; SimpleElementPartition(h, numberToReadPerProc, startingIndex, el_start, el_end, numElements); std::vector vertexIDs; - if (numberToReadPerProc[PCU_Comm_Self2(h)] > 0) - vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self2(h)] * verticesPerElement, + if (numberToReadPerProc[PCU_Comm_Self(h)] > 0) + vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self(h)] * verticesPerElement, -1234567); - const auto start = startingIndex[PCU_Comm_Self2(h)]; - const auto end = startingIndex[PCU_Comm_Self2h(h)] + numberToReadPerProc[PCU_Comm_Self2(h)] - 1; - DebugParallelPrinter(h, std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self2(h)]); + const auto start = startingIndex[PCU_Comm_Self(h)]; + const auto end = startingIndex[PCU_Comm_Self(h)] + numberToReadPerProc[PCU_Comm_Self(h)] - 1; + DebugParallelPrinter(h, std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self(h)]); // cgp_elements_read_data(cgid, base, zone, section, start, end, vertexIDs.data()); - if (numberToReadPerProc[PCU_Comm_Self2(h)] > 0) + if (numberToReadPerProc[PCU_Comm_Self(h)] > 0) { localElementRanges.push_back(std::make_pair(start, end)); } @@ -419,7 +419,7 @@ auto ReadElements(PCUHandle h, int cgid, int base, int zone, int section, int el i = i - 1; } - return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self2(h)]); + return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self(h)]); } struct CGNSBCMeta @@ -856,7 +856,7 @@ struct BCInfo } }; // namespace -void ReadBCInfo(PCUHandle h, const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector &bcInfos, const apf::GlobalToVert &globalToVert) +void ReadBCInfo(PCU_t h, const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector &bcInfos, const apf::GlobalToVert &globalToVert) { // Read the BCS. std::vector bcMetas(nBocos); @@ -1049,12 +1049,12 @@ void ReadBCInfo(PCUHandle h, const int cgid, const int base, const int zone, con } } -apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &readMeshData) +apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &readMeshData) { static_assert(std::is_same::value, "cgsize_t not compiled as int"); int cgid = -1; - auto comm = PCU_Get_Comm2(h); + auto comm = PCU_Get_Comm(h); cgp_mpi_comm(comm); cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); cgp_open(fname.c_str(), CGNS_ENUMV(CG_MODE_READ), &cgid); @@ -1632,7 +1632,7 @@ apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSB } // free up memory - if (PCU_Comm_Initialized2(h)) + if (PCU_Comm_Initialized(h)) cgp_close(cgid); else cg_close(cgid); @@ -1711,7 +1711,7 @@ apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSB return mesh; } // namespace -apf::Mesh2 *DoIt(PCUHandle h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap) +apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap) { std::vector> meshData; return DoIt(h, g, fname, cgnsBCMap, meshData); @@ -1725,11 +1725,11 @@ namespace apf // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) { - PCUHandle h = PCU_Get_Global_Handle(); - return loadMdsFromCGNS2(h, g, fname, cgnsBCMap, meshData); + PCU_t h = PCU_Get_Global_Handle(); + return loadMdsFromCGNS(h, g, fname, cgnsBCMap, meshData); } -Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) +Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) { #ifdef HAVE_CGNS Mesh2 *m = DoIt(h, g, fname, cgnsBCMap, meshData); @@ -1746,11 +1746,11 @@ Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSB // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) { - PCUHandle h = PCU_Get_Global_Handle(); - return loadMdsFromCGNS2(h, g, fname, cgnsBCMap); + PCU_t h = PCU_Get_Global_Handle(); + return loadMdsFromCGNS(h, g, fname, cgnsBCMap); } -Mesh2 *loadMdsFromCGNS2(PCUHandle h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) +Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) { #ifdef HAVE_CGNS Mesh2 *m = DoIt(h, g, fname, cgnsBCMap); diff --git a/mds/mds_apf.c b/mds/mds_apf.c index 1a7903016..92f6db313 100644 --- a/mds/mds_apf.c +++ b/mds/mds_apf.c @@ -11,7 +11,7 @@ #include "mds_apf.h" #include #include -#include +#include struct mds_apf* mds_apf_create(struct gmi_model* model, int d, mds_id cap[MDS_TYPES]) @@ -148,17 +148,17 @@ int mds_model_id(struct mds_apf* m, struct gmi_ent* model) return gmi_tag(m->user_model, model); } -static void downs_to_copy(PCUHandle h, struct mds_set* s, +static void downs_to_copy(PCU_t h, struct mds_set* s, struct mds_copy c) { int i; - PCU_COMM_PACK2(h, c.p, c.e); + PCU_COMM_PACK(h, c.p, c.e); for (i = 0; i < s->n; ++i) - PCU_COMM_PACK2(h, c.p, s->e[i]); + PCU_COMM_PACK(h, c.p, s->e[i]); } static void downs_to_copies( - PCUHandle h, struct mds* m, mds_id e, struct mds_copies* c) + PCU_t h, struct mds* m, mds_id e, struct mds_copies* c) { int i; struct mds_set s; @@ -219,19 +219,19 @@ static void rotate_set(struct mds_set* in, int r, struct mds_set* out) out->e[i] = in->e[(i + r) % in->n]; } -static int recv_down_copies(PCUHandle h, struct mds_net* net, struct mds* m) +static int recv_down_copies(PCU_t h, struct mds_net* net, struct mds* m) { mds_id e; struct mds_set s; struct mds_set rs; struct mds_set s2; int i; - int from = PCU_Comm_Sender2(h); - PCU_COMM_UNPACK2(h, e); + int from = PCU_Comm_Sender(h); + PCU_COMM_UNPACK(h, e); mds_get_adjacent(m, e, mds_dim[mds_type(e)] - 1, &s); rs.n = s.n; for (i = 0; i < s.n; ++i) - PCU_COMM_UNPACK2(h, rs.e[i]); + PCU_COMM_UNPACK(h, rs.e[i]); if (compare_copy_sets(net, &s, from, &rs)) return 0; for (i = -s.n; i < s.n; ++i) { @@ -251,26 +251,26 @@ static int copy_less(struct mds_copy a, struct mds_copy b) return a.e < b.e; } -static int owns_copies(PCUHandle h, mds_id e, struct mds_copies* c) +static int owns_copies(PCU_t h, mds_id e, struct mds_copies* c) { int i; struct mds_copy mc; mc.e = e; - mc.p = PCU_Comm_Self2(h); + mc.p = PCU_Comm_Self(h); for (i = 0; i < c->n; ++i) if (copy_less(c->c[i], mc)) return 0; return 1; } -static int align_copies(PCUHandle h, struct mds_net* net, struct mds* m) +static int align_copies(PCU_t h, struct mds_net* net, struct mds* m) { int d; mds_id e; struct mds_copies* c; int did_change = 0; for (d = 1; d < m->d; ++d){ - PCU_Comm_Begin2(h); + PCU_Comm_Begin(h); for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) { c = mds_get_copies(net, e); if (!c) @@ -278,28 +278,28 @@ static int align_copies(PCUHandle h, struct mds_net* net, struct mds* m) if (owns_copies(h, e, c)) downs_to_copies(h, m, e, c); } - PCU_Comm_Send2(h); - while (PCU_Comm_Receive2(h)) + PCU_Comm_Send(h); + while (PCU_Comm_Receive(h)) if (recv_down_copies(h, net, m)) did_change = 1; } - return PCU_Or2(h, did_change); + return PCU_Or(h, did_change); } -int mds_align_matches(PCUHandle h, struct mds_apf* m) +int mds_align_matches(PCU_t h, struct mds_apf* m) { return align_copies(h, &m->matches, &m->mds); } // seol -int mds_align_ghosts(PCUHandle h, struct mds_apf* m) +int mds_align_ghosts(PCU_t h, struct mds_apf* m) { return align_copies(h, &m->ghosts, &m->mds); } -int mds_align_remotes(PCUHandle h, struct mds_apf* m) +int mds_align_remotes(PCU_t h, struct mds_apf* m) { return align_copies(h, &m->remotes, &m->mds); } diff --git a/mds/mds_apf.h b/mds/mds_apf.h index 38bebf423..65fff23d9 100644 --- a/mds/mds_apf.h +++ b/mds/mds_apf.h @@ -26,7 +26,7 @@ struct pcu_file; struct gmi_model; struct gmi_ent; -typedef struct PCUHandle PCUHandle; +typedef struct PCU_t PCU_t; struct mds_apf { struct mds mds; @@ -57,24 +57,24 @@ void* mds_get_part(struct mds_apf* m, mds_id e); void mds_set_part(struct mds_apf* m, mds_id e, void* p); struct mds_tag* mds_number_verts_bfs(struct mds_apf* m); -struct mds_apf* mds_reorder(PCUHandle h, struct mds_apf* m, int ignore_peers, +struct mds_apf* mds_reorder(PCU_t h, struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers); struct gmi_ent* mds_find_model(struct mds_apf* m, int dim, int id); int mds_model_dim(struct mds_apf* m, struct gmi_ent* model); int mds_model_id(struct mds_apf* m, struct gmi_ent* model); -struct mds_apf* mds_read_smb(PCUHandle h, struct gmi_model* model, const char* pathname, +struct mds_apf* mds_read_smb(PCU_t h, struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh); -struct mds_apf* mds_write_smb(PCUHandle h, struct mds_apf* m, const char* pathname, +struct mds_apf* mds_write_smb(PCU_t h, struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh); void mds_verify(struct mds_apf* m); void mds_verify_residence(struct mds_apf* m, mds_id e); -int mds_align_matches(PCUHandle h, struct mds_apf* m); -int mds_align_ghosts(PCUHandle h, struct mds_apf* m); -int mds_align_remotes(PCUHandle h, struct mds_apf* m); +int mds_align_matches(PCU_t h, struct mds_apf* m); +int mds_align_ghosts(PCU_t h, struct mds_apf* m); +int mds_align_remotes(PCU_t h, struct mds_apf* m); void mds_derive_model(struct mds_apf* m); void mds_update_model_for_entity(struct mds_apf* m, mds_id e, diff --git a/mds/mds_net.c b/mds/mds_net.c index 83cb83271..1e8dd9ce5 100644 --- a/mds/mds_net.c +++ b/mds/mds_net.c @@ -9,7 +9,7 @@ *******************************************************************************/ #include "mds_net.h" -#include +#include #include #include #include @@ -159,8 +159,8 @@ static void note_peer(struct mds_links* ln, int p) ++(ln->n[i]); } -static void for_type_net(PCUHandle h, struct mds_net* net, struct mds* m, - int t, void (*f)(PCUHandle h, mds_id i, struct mds_copy c, void* u), void* u) +static void for_type_net(PCU_t h, struct mds_net* net, struct mds* m, + int t, void (*f)(PCU_t h, mds_id i, struct mds_copy c, void* u), void* u) { mds_id i; int j; @@ -174,10 +174,10 @@ static void for_type_net(PCUHandle h, struct mds_net* net, struct mds* m, } } -static void note_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) +static void note_remote_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { (void)i; - if (c.p != PCU_Comm_Self2(h)) + if (c.p != PCU_Comm_Self(h)) note_peer(u, c.p); } @@ -189,11 +189,11 @@ static void alloc_links(struct mds_links* ln) ln->l[i] = malloc(ln->n[i] * sizeof(unsigned)); } -static void take_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) +static void take_remote_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; int pi; - if (PCU_Comm_Self2(h) < c.p) { + if (PCU_Comm_Self(h) < c.p) { pi = find_peer(ln, c.p); ln->l[pi][ln->n[pi]] = i; /* use n as position keeper */ @@ -201,28 +201,28 @@ static void take_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) } } -static void send_remote_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) +static void send_remote_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { (void)i; (void)u; - if (PCU_Comm_Self2(h) < c.p) - PCU_COMM_PACK2(h, c.p, c.e); + if (PCU_Comm_Self(h) < c.p) + PCU_COMM_PACK(h, c.p, c.e); } -static void recv_links(PCUHandle h, struct mds_links* ln) +static void recv_links(PCU_t h, struct mds_links* ln) { int from; mds_id* tmp; int pi; unsigned i; - from = PCU_Comm_Sender2(h); + from = PCU_Comm_Sender(h); pi = find_peer(ln, from); - tmp = PCU_Comm_Extract2(h, ln->n[pi] * sizeof(mds_id)); + tmp = PCU_Comm_Extract(h, ln->n[pi] * sizeof(mds_id)); for (i = 0; i < ln->n[pi]; ++i) ln->l[pi][i] = mds_index(tmp[i]); } -void mds_get_type_links(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_get_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; @@ -230,39 +230,39 @@ void mds_get_type_links(PCUHandle h, struct mds_net* net, struct mds* m, for_type_net(h, net, m, t, note_remote_link, ln); alloc_links(ln); for (i = 0; i < ln->np; ++i) - if (((unsigned)PCU_Comm_Self2(h)) < ln->p[i]) + if (((unsigned)PCU_Comm_Self(h)) < ln->p[i]) /* zero n's for behavior in take_copy */ ln->n[i] = 0; /* record indices in local order for owned boundaries */ for_type_net(h, net, m, t, take_remote_link, ln); - PCU_Comm_Begin2(h); + PCU_Comm_Begin(h); /* send indices in local order for owned boundaries */ for_type_net(h, net, m, t, send_remote_link, NULL); - PCU_Comm_Send2(h); - while (PCU_Comm_Listen2(h)) + PCU_Comm_Send(h); + while (PCU_Comm_Listen(h)) /* recv indices in remote order for non-owned boundaries */ recv_links(h, ln); } -void mds_set_type_links(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; unsigned j; unsigned* in; struct mds_copy c; - PCU_Comm_Begin2(h); + PCU_Comm_Begin(h); for (i = 0; i < ln->np; ++i) { PCU_ALWAYS_ASSERT(ln->l); for (j = 0; j < ln->n[i]; ++j) - PCU_COMM_PACK2(h, ln->p[i], ln->l[i][j]); + PCU_COMM_PACK(h, ln->p[i], ln->l[i][j]); } - PCU_Comm_Send2(h); - while (PCU_Comm_Listen2(h)) { - c.p = PCU_Comm_Sender2(h); - PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self2(h)); + PCU_Comm_Send(h); + while (PCU_Comm_Listen(h)) { + c.p = PCU_Comm_Sender(h); + PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self(h)); i = find_peer(ln, c.p); - in = PCU_Comm_Extract2(h, ln->n[i] * sizeof(unsigned)); + in = PCU_Comm_Extract(h, ln->n[i] * sizeof(unsigned)); for (j = 0; j < ln->n[i]; ++j) { c.e = mds_identify(t, in[j]); mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c); @@ -289,23 +289,23 @@ int mds_net_empty(struct mds_net* net) return 1; } -static void note_local_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) +static void note_local_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { - if (c.p == PCU_Comm_Self2(h)) { + if (c.p == PCU_Comm_Self(h)) { if (i < mds_index(c.e)) - note_peer(u, PCU_Comm_Self2(h)); + note_peer(u, PCU_Comm_Self(h)); else /* hack id to store self-receivers */ - note_peer(u, PCU_Comm_Peers2(h)); + note_peer(u, PCU_Comm_Peers(h)); } } -static void take_local_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) +static void take_local_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; - int self = find_peer(ln, PCU_Comm_Self2(h)); - int other = find_peer(ln, PCU_Comm_Peers2(h)); + int self = find_peer(ln, PCU_Comm_Self(h)); + int other = find_peer(ln, PCU_Comm_Peers(h)); mds_id j = mds_index(c.e); - if ((PCU_Comm_Self2(h) == c.p) && (i < j)) { + if ((PCU_Comm_Self(h) == c.p) && (i < j)) { ln->l[self][ln->n[self]] = i; ln->l[other][ln->n[other]] = j; /* use ns as (redundant) position keepers */ @@ -314,15 +314,15 @@ static void take_local_link(PCUHandle h, mds_id i, struct mds_copy c, void* u) } } -void mds_get_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_get_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; for_type_net(h, net, m, t, note_local_link, ln); - self = find_peer(ln, PCU_Comm_Self2(h)); + self = find_peer(ln, PCU_Comm_Self(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers2(h)); + other = find_peer(ln, PCU_Comm_Peers(h)); PCU_ALWAYS_ASSERT(ln->n[self] == ln->n[other]); ln->l[self] = malloc(ln->n[self] * sizeof(unsigned)); ln->l[other] = malloc(ln->n[other] * sizeof(unsigned)); @@ -331,18 +331,18 @@ void mds_get_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, for_type_net(h, net, m, t, take_local_link, ln); } -void mds_set_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; unsigned i; mds_id a, b; struct mds_copy c; - c.p = PCU_Comm_Self2(h); - self = find_peer(ln, PCU_Comm_Self2(h)); + c.p = PCU_Comm_Self(h); + self = find_peer(ln, PCU_Comm_Self(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers2(h)); + other = find_peer(ln, PCU_Comm_Peers(h)); PCU_ALWAYS_ASSERT(ln->n != 0); PCU_ALWAYS_ASSERT(ln->n[self] == ln->n[other]); for (i = 0; i < ln->n[self]; ++i) { @@ -356,13 +356,13 @@ void mds_set_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, } } -void mds_free_local_links(PCUHandle h, struct mds_links* ln) +void mds_free_local_links(PCU_t h, struct mds_links* ln) { int self, other; - self = find_peer(ln, PCU_Comm_Self2(h)); + self = find_peer(ln, PCU_Comm_Self(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers2(h)); + other = find_peer(ln, PCU_Comm_Peers(h)); PCU_ALWAYS_ASSERT(ln->n != 0); ln->n[self] = ln->n[other] = 0; PCU_ALWAYS_ASSERT(ln->l != 0); diff --git a/mds/mds_net.h b/mds/mds_net.h index a1e6da1e8..8cc7de09d 100644 --- a/mds/mds_net.h +++ b/mds/mds_net.h @@ -13,7 +13,7 @@ #include "mds.h" -typedef struct PCUHandle PCUHandle; +typedef struct PCU_t PCU_t; struct mds_copy { mds_id e; @@ -52,18 +52,18 @@ void mds_grow_net( void mds_add_copy(struct mds_net* net, struct mds* m, mds_id e, struct mds_copy c); -void mds_get_type_links(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_get_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_set_type_links(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); void mds_free_links(struct mds_links* ln); int mds_net_empty(struct mds_net* net); -void mds_get_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_get_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_set_local_matches(PCUHandle h, struct mds_net* net, struct mds* m, +void mds_set_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_free_local_links(PCUHandle h, struct mds_links* ln); +void mds_free_local_links(PCU_t h, struct mds_links* ln); #endif diff --git a/mds/mds_order.c b/mds/mds_order.c index b19c3a389..e02d51f07 100644 --- a/mds/mds_order.c +++ b/mds/mds_order.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include struct queue { mds_id* e; @@ -183,7 +183,7 @@ static mds_id lookup(struct mds_tag* tag, mds_id old) } /* see apf/apfConvert.cc apf::Converter::createRemotes */ -static void rebuild_net(PCUHandle h, struct mds_net* net, +static void rebuild_net(PCU_t h, struct mds_net* net, struct mds* m, struct mds_net* net2, struct mds* m2, @@ -197,7 +197,7 @@ static void rebuild_net(PCUHandle h, struct mds_net* net, struct mds_copies* cs; struct mds_copy c; int i; - PCU_Comm_Begin2(h); + PCU_Comm_Begin(h); for (d = 0; d <= m->d; ++d) for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) { cs = mds_get_copies(net, e); @@ -206,16 +206,16 @@ static void rebuild_net(PCUHandle h, struct mds_net* net, ne = lookup(new_of, e); for (i = 0; i < cs->n; ++i) { ce = cs->c[i].e; - PCU_COMM_PACK2(h, cs->c[i].p, ce); - PCU_COMM_PACK2(h, cs->c[i].p, ne); + PCU_COMM_PACK(h, cs->c[i].p, ce); + PCU_COMM_PACK(h, cs->c[i].p, ne); } } - PCU_Comm_Send2(h); - while (PCU_Comm_Listen2(h)) { - c.p = PCU_Comm_Sender2(h); - while (!PCU_Comm_Unpacked2(h)) { - PCU_COMM_UNPACK2(h, ce); - PCU_COMM_UNPACK2(h, ne); + PCU_Comm_Send(h); + while (PCU_Comm_Listen(h)) { + c.p = PCU_Comm_Sender(h); + while (!PCU_Comm_Unpacked(h)) { + PCU_COMM_UNPACK(h, ce); + PCU_COMM_UNPACK(h, ne); c.e = ne; nce = lookup(new_of, ce); mds_add_copy(net2, m2, nce, c); @@ -380,7 +380,7 @@ static void rebuild_parts( } static struct mds_apf* rebuild( - PCUHandle h, + PCU_t h, struct mds_apf* m, struct mds_tag* new_of, int ignore_peers) @@ -406,7 +406,7 @@ static struct mds_apf* rebuild( return m2; } -struct mds_apf* mds_reorder(PCUHandle h, struct mds_apf* m, int ignore_peers, +struct mds_apf* mds_reorder(PCU_t h, struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers) { struct mds_tag* tag; diff --git a/mds/mds_smb.c b/mds/mds_smb.c index edc6dc82d..3c96c2d07 100644 --- a/mds/mds_smb.c +++ b/mds/mds_smb.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -51,7 +51,7 @@ enum { #define MAX_PEERS (10*1000) #define MAX_TAGS (100) -typedef struct PCUHandle PCUHandle; +typedef struct PCU_t PCU_t; static int smb2mds(int smb_type) { @@ -118,7 +118,7 @@ static void write_links(struct pcu_file* f, struct mds_links* l) pcu_write_unsigneds(f, l->l[i], l->n[i]); } -static void read_header(PCUHandle h, struct pcu_file* f, unsigned* version, unsigned* dim, +static void read_header(PCU_t h, struct pcu_file* f, unsigned* version, unsigned* dim, int ignore_peers) { unsigned magic, np; @@ -128,12 +128,12 @@ static void read_header(PCUHandle h, struct pcu_file* f, unsigned* version, unsi PCU_READ_UNSIGNED(f, *dim); PCU_READ_UNSIGNED(f, np); if (*version >= 1 && (!ignore_peers)) - if (np != (unsigned)PCU_Comm_Peers2(h)) + if (np != (unsigned)PCU_Comm_Peers(h)) reel_fail("To whom it may concern\n" "the # of mesh partitions != the # of MPI ranks"); } -static void write_header(PCUHandle h, struct pcu_file* f, unsigned dim, +static void write_header(PCU_t h, struct pcu_file* f, unsigned dim, int ignore_peers) { unsigned magic = 0; @@ -145,7 +145,7 @@ static void write_header(PCUHandle h, struct pcu_file* f, unsigned dim, if (ignore_peers) np = 1; else - np = (unsigned)PCU_Comm_Peers2(h); + np = (unsigned)PCU_Comm_Peers(h); PCU_WRITE_UNSIGNED(f, np); } @@ -212,7 +212,7 @@ static void write_conn(struct pcu_file* f, struct mds_apf* m) } } -static void read_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, +static void read_remotes(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; @@ -222,7 +222,7 @@ static void read_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, mds_free_links(&ln); } -static void write_remotes(PCUHandle h, struct pcu_file* f, struct mds_apf* m, +static void write_remotes(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; @@ -510,7 +510,7 @@ static void write_tags(struct pcu_file* f, struct mds_apf* m) free(sizes); } -static void read_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int t, +static void read_type_matches(PCU_t h, struct pcu_file* f, struct mds_apf* m, int t, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; @@ -523,7 +523,7 @@ static void read_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m mds_free_links(&ln); } -static void write_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, int t, +static void write_type_matches(PCU_t h, struct pcu_file* f, struct mds_apf* m, int t, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; @@ -535,7 +535,7 @@ static void write_type_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* mds_free_links(&ln); } -static void read_matches_old(PCUHandle h, struct pcu_file* f, struct mds_apf* m, +static void read_matches_old(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; @@ -543,7 +543,7 @@ static void read_matches_old(PCUHandle h, struct pcu_file* f, struct mds_apf* m, read_type_matches(h, f, m, t, ignore_peers); } -static void read_matches_new(PCUHandle h, struct pcu_file* f, struct mds_apf* m, +static void read_matches_new(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; @@ -551,7 +551,7 @@ static void read_matches_new(PCUHandle h, struct pcu_file* f, struct mds_apf* m, read_type_matches(h, f, m, smb2mds(t), ignore_peers); } -static void write_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, +static void write_matches(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; @@ -559,7 +559,7 @@ static void write_matches(PCUHandle h, struct pcu_file* f, struct mds_apf* m, write_type_matches(h, f, m, smb2mds(t), ignore_peers); } -static struct mds_apf* read_smb(PCUHandle h, struct gmi_model* model, const char* filename, +static struct mds_apf* read_smb(PCU_t h, struct gmi_model* model, const char* filename, int zip, int ignore_peers, void* apf_mesh) { struct mds_apf* m; @@ -614,7 +614,7 @@ static void write_coords(struct pcu_file* f, struct mds_apf* m) pcu_write_doubles(f, &m->param[0][0], count); } -static void write_smb(PCUHandle h, struct mds_apf* m, const char* filename, +static void write_smb(PCU_t h, struct mds_apf* m, const char* filename, int zip, int ignore_peers, void* apf_mesh) { struct pcu_file* f; @@ -688,7 +688,7 @@ static void safe_mkdir(const char* path, mode_t mode) reel_fail("MDS: could not create directory \"%s\"\n", path); } -static char* handle_path(PCUHandle h, const char* in, int is_write, int* zip, +static char* handle_path(PCU_t h, const char* in, int is_write, int* zip, int ignore_peers) { static const char* zippre = "bz2:"; @@ -696,7 +696,7 @@ static char* handle_path(PCUHandle h, const char* in, int is_write, int* zip, size_t bufsize; char* path; mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - int self = PCU_Comm_Self2(h); + int self = PCU_Comm_Self(h); bufsize = strlen(in) + 256; path = malloc(bufsize); strcpy(path, in); @@ -712,14 +712,14 @@ static char* handle_path(PCUHandle h, const char* in, int is_write, int* zip, if (is_write) { if (!self) safe_mkdir(path, dir_perm); - PCU_Barrier2(h); + PCU_Barrier(h); } - if (PCU_Comm_Peers2(h) > SMB_FANOUT) { + if (PCU_Comm_Peers(h) > SMB_FANOUT) { append(path, bufsize, "%d/", self / SMB_FANOUT); if (is_write) { if (self % SMB_FANOUT == 0) safe_mkdir(path, dir_perm); - PCU_Barrier2(h); + PCU_Barrier(h); } } } else if (ends_with(path, smbext)) { @@ -731,7 +731,7 @@ static char* handle_path(PCUHandle h, const char* in, int is_write, int* zip, return path; } -struct mds_apf* mds_read_smb(PCUHandle h, struct gmi_model* model, const char* pathname, +struct mds_apf* mds_read_smb(PCU_t h, struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh) { char* filename; @@ -752,18 +752,18 @@ static int is_compact(struct mds_apf* m) return 1; } -struct mds_apf* mds_write_smb(PCUHandle h, struct mds_apf* m, const char* pathname, +struct mds_apf* mds_write_smb(PCU_t h, struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh) { const char* reorderWarning ="MDS: reordering before writing smb files\n"; char* filename; int zip; if (ignore_peers && (!is_compact(m))) { - if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); + if(!PCU_Comm_Self(h)) lion_eprint(1, "%s", reorderWarning); m = mds_reorder(h, m, 1, mds_number_verts_bfs(m)); } - if ((!ignore_peers) && PCU_Or2(h, !is_compact(m))) { - if(!PCU_Comm_Self2(h)) lion_eprint(1, "%s", reorderWarning); + if ((!ignore_peers) && PCU_Or(h, !is_compact(m))) { + if(!PCU_Comm_Self(h)) lion_eprint(1, "%s", reorderWarning); m = mds_reorder(h, m, 0, mds_number_verts_bfs(m)); } filename = handle_path(h, pathname, 1, &zip, ignore_peers); diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index d91114514..927a52ef1 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,8 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - #pcu.cc - pcu2.cc + pcu.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -27,12 +26,12 @@ set(SOURCES # Package headers set(HEADERS - #PCU.h - PCU2.h + PCU.h pcu_io.h pcu_util.h reel/reel.h pcu_defines.h + PCUObj.h ) # Add the pcu library diff --git a/pcu/PCU.h b/pcu/PCU.h new file mode 100644 index 000000000..a954832a9 --- /dev/null +++ b/pcu/PCU.h @@ -0,0 +1,120 @@ +#ifndef PCU_H +#define PCU_H +#include "pcu_defines.h" +#include + +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif + +typedef struct PCU_t PCU_t; + +int PCU_Comm_Init(PCU_t* h); +int PCU_Comm_Free(PCU_t* h); + +/*rank/size functions*/ +int PCU_Comm_Self(PCU_t h); +int PCU_Comm_Peers(PCU_t h); + +/*recommended message passing API*/ +void PCU_Comm_Begin(PCU_t h); +int PCU_Comm_Pack(PCU_t h, int to_rank, const void* data, size_t size); +#define PCU_COMM_PACK(handle, to_rank,object)\ +PCU_Comm_Pack(handle, to_rank,&(object),sizeof(object)) +int PCU_Comm_Send(PCU_t h); +bool PCU_Comm_Receive(PCU_t h); +bool PCU_Comm_Listen(PCU_t h); +int PCU_Comm_Sender(PCU_t h); +bool PCU_Comm_Unpacked(PCU_t h); +int PCU_Comm_Unpack(PCU_t h, void* data, size_t size); +#define PCU_COMM_UNPACK(handle, object)\ +PCU_Comm_Unpack(handle, &(object),sizeof(object)) + +/*turns deterministic ordering for the + above API on/off*/ +void PCU_Comm_Order(PCU_t h, bool on); + +/*collective operations*/ +void PCU_Barrier(PCU_t h); +void PCU_Add_Doubles(PCU_t h, double* p, size_t n); +double PCU_Add_Double(PCU_t h, double x); +void PCU_Min_Doubles(PCU_t h, double* p, size_t n); +double PCU_Min_Double(PCU_t h, double x); +void PCU_Max_Doubles(PCU_t h, double* p, size_t n); +double PCU_Max_Double(PCU_t h, double x); +void PCU_Add_Ints(PCU_t h, int* p, size_t n); +int PCU_Add_Int(PCU_t h, int x); +void PCU_Add_Longs(PCU_t h, long* p, size_t n); +long PCU_Add_Long(PCU_t h, long x); +void PCU_Exscan_Ints(PCU_t h, int* p, size_t n); +int PCU_Exscan_Int(PCU_t h, int x); +void PCU_Exscan_Longs(PCU_t h, long* p, size_t n); +long PCU_Exscan_Long(PCU_t h, long x); +void PCU_Add_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Add_SizeT(PCU_t h, size_t x); +void PCU_Min_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Min_SizeT(PCU_t h, size_t x); +void PCU_Max_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Max_SizeT(PCU_t h, size_t x); +void PCU_Min_Ints(PCU_t h, int* p, size_t n); +int PCU_Min_Int(PCU_t h, int x); +void PCU_Max_Ints(PCU_t h, int* p, size_t n); +int PCU_Max_Int(PCU_t h, int x); +void PCU_Max_Longs(PCU_t h, long* p, size_t n); +long PCU_Max_Long(PCU_t h, long x); +int PCU_Or(PCU_t h, int c); +int PCU_And(PCU_t h, int c); + +/*process-level self/peers (mpi wrappers)*/ +int PCU_Proc_Self(PCU_t h); +int PCU_Proc_Peers(PCU_t h); + +/*IPComMan replacement API*/ +int PCU_Comm_Write(PCU_t h, int to_rank, const void* data, size_t size); +#define PCU_COMM_WRITE(handle,to,data) \ +PCU_Comm_Write(handle, to,&(data),sizeof(data)) +bool PCU_Comm_Read(PCU_t h, int* from_rank, void** data, size_t* size); + +/*Debug file I/O API*/ +void PCU_Debug_Open(PCU_t h); + +void PCU_Debug_Print(PCU_t h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(2,3); +/*lesser-used APIs*/ +bool PCU_Comm_Initialized(PCU_t h); +int PCU_Comm_Packed(PCU_t h ,int to_rank, size_t* size); +int PCU_Comm_From(PCU_t h, int* from_rank); +int PCU_Comm_Received(PCU_t h, size_t* size); +void* PCU_Comm_Extract(PCU_t h, size_t size); +int PCU_Comm_Rank(PCU_t h, int* rank); +int PCU_Comm_Size(PCU_t h, int* size); + +/*deprecated method enum*/ + +/*special MPI_Comm replacement API*/ +void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm); +MPI_Comm PCU_Get_Comm(PCU_t h); + +/*stack trace helpers using GNU/Linux*/ +void PCU_Protect(void); + +/*MPI_Wtime() equivalent*/ +double PCU_Time(void); + +/*Memory usage*/ +double PCU_GetMem(void); + +/*Access global variable*/ +PCU_t PCU_Get_Global_Handle(void); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif \ No newline at end of file diff --git a/pcu/PCU2.h b/pcu/PCU2.h deleted file mode 100644 index 914e25299..000000000 --- a/pcu/PCU2.h +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef PCU2_H -#define PCU2_H -#include "pcu_defines.h" -#include - -#ifdef __cplusplus -#include -#include -extern "C" { -#else -#include -#include -#include -#endif - -typedef struct PCUHandle PCUHandle; - -int PCU_Comm_Init2(PCUHandle* h); -int PCU_Comm_Free2(PCUHandle* h); - -/*rank/size functions*/ -int PCU_Comm_Self2(PCUHandle h); -int PCU_Comm_Peers2(PCUHandle h); - -/*recommended message passing API*/ -void PCU_Comm_Begin2(PCUHandle h); -int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void* data, size_t size); -#define PCU_COMM_PACK2(handle, to_rank,object)\ -PCU_Comm_Pack2(handle, to_rank,&(object),sizeof(object)) -int PCU_Comm_Send2(PCUHandle h); -bool PCU_Comm_Receive2(PCUHandle h); -bool PCU_Comm_Listen2(PCUHandle h); -int PCU_Comm_Sender2(PCUHandle h); -bool PCU_Comm_Unpacked2(PCUHandle h); -int PCU_Comm_Unpack2(PCUHandle h, void* data, size_t size); -#define PCU_COMM_UNPACK2(handle, object)\ -PCU_Comm_Unpack2(handle, &(object),sizeof(object)) - -/*turns deterministic ordering for the - above API on/off*/ -void PCU_Comm_Order2(PCUHandle h, bool on); - -/*collective operations*/ -void PCU_Barrier2(PCUHandle h); -void PCU_Add_Doubles2(PCUHandle h, double* p, size_t n); -double PCU_Add_Double2(PCUHandle h, double x); -void PCU_Min_Doubles2(PCUHandle h, double* p, size_t n); -double PCU_Min_Double2(PCUHandle h, double x); -void PCU_Max_Doubles2(PCUHandle h, double* p, size_t n); -double PCU_Max_Double2(PCUHandle h, double x); -void PCU_Add_Ints2(PCUHandle h, int* p, size_t n); -int PCU_Add_Int2(PCUHandle h, int x); -void PCU_Add_Longs2(PCUHandle h, long* p, size_t n); -long PCU_Add_Long2(PCUHandle h, long x); -void PCU_Exscan_Ints2(PCUHandle h, int* p, size_t n); -int PCU_Exscan_Int2(PCUHandle h, int x); -void PCU_Exscan_Longs2(PCUHandle h, long* p, size_t n); -long PCU_Exscan_Long2(PCUHandle h, long x); -void PCU_Add_SizeTs2(PCUHandle h, size_t* p, size_t n); -size_t PCU_Add_SizeT2(PCUHandle h, size_t x); -void PCU_Min_SizeTs2(PCUHandle h, size_t* p, size_t n); -size_t PCU_Min_SizeT2(PCUHandle h, size_t x); -void PCU_Max_SizeTs2(PCUHandle h, size_t* p, size_t n); -size_t PCU_Max_SizeT2(PCUHandle h, size_t x); -void PCU_Min_Ints2(PCUHandle h, int* p, size_t n); -int PCU_Min_Int2(PCUHandle h, int x); -void PCU_Max_Ints2(PCUHandle h, int* p, size_t n); -int PCU_Max_Int2(PCUHandle h, int x); -void PCU_Max_Longs2(PCUHandle h, long* p, size_t n); -long PCU_Max_Long2(PCUHandle h, long x); -int PCU_Or2(PCUHandle h, int c); -int PCU_And2(PCUHandle h, int c); - -/*process-level self/peers (mpi wrappers)*/ -int PCU_Proc_Self2(PCUHandle h); -int PCU_Proc_Peers2(PCUHandle h); - -/*IPComMan replacement API*/ -int PCU_Comm_Write2(PCUHandle h, int to_rank, const void* data, size_t size); -#define PCU_COMM_WRITE2(handle,to,data) \ -PCU_Comm_Write2(handle, to,&(data),sizeof(data)) -bool PCU_Comm_Read2(PCUHandle h, int* from_rank, void** data, size_t* size); - -/*Debug file I/O API*/ -void PCU_Debug_Open2(PCUHandle h); - -void PCU_Debug_Print2(PCUHandle h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(2,3); -/*lesser-used APIs*/ -bool PCU_Comm_Initialized2(PCUHandle h); -int PCU_Comm_Packed2(PCUHandle h ,int to_rank, size_t* size); -int PCU_Comm_From2(PCUHandle h, int* from_rank); -int PCU_Comm_Received2(PCUHandle h, size_t* size); -void* PCU_Comm_Extract2(PCUHandle h, size_t size); -int PCU_Comm_Rank2(PCUHandle h, int* rank); -int PCU_Comm_Size2(PCUHandle h, int* size); - -/*deprecated method enum*/ - -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm2(PCUHandle h); - -/*stack trace helpers using GNU/Linux*/ -void PCU_Protect2(void); - -/*MPI_Wtime() equivalent*/ -double PCU_Time2(void); - -/*Memory usage*/ -double PCU_GetMem2(void); - -/*Access global variable*/ -PCUHandle PCU_Get_Global_Handle(void); - - - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif \ No newline at end of file diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h index 723b6c048..307e051b6 100644 --- a/pcu/PCUObj.h +++ b/pcu/PCUObj.h @@ -27,7 +27,7 @@ class PCU { [[nodiscard]] int Peers() const noexcept; [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; - [[nodiscard]] PCUHandle GetCHandle() {PCUHandle h; h.ptr=this; return h;} + [[nodiscard]] PCU_t GetCHandle() {PCU_t h; h.ptr=this; return h;} /*recommended message passing API*/ void Begin() noexcept; int Pack(int to_rank, const void *data, size_t size) noexcept; diff --git a/pcu/pcu2.cc b/pcu/pcu.cc similarity index 82% rename from pcu/pcu2.cc rename to pcu/pcu.cc index 593cc0bbf..e1f1a7feb 100644 --- a/pcu/pcu2.cc +++ b/pcu/pcu.cc @@ -1,4 +1,4 @@ -#include "PCU2.h" +#include "PCU.h" #include "PCUObj.h" #include "reel.h" #include @@ -7,16 +7,15 @@ extern "C" { -int PCU_Comm_Init2(PCUHandle* h) { +int PCU_Comm_Init(PCU_t* h) { if (h->ptr != nullptr) reel_fail("nested calls to Comm_Init"); pcu::PCU* pcu_object = new pcu::PCU(MPI_COMM_WORLD); - //PCUHandle h; h->ptr = static_cast(pcu_object); return PCU_SUCCESS; } -int PCU_Comm_Free2(PCUHandle* h) { +int PCU_Comm_Free(PCU_t* h) { if (h->ptr == nullptr) reel_fail("Comm_Free called before Comm_Init"); delete static_cast(h->ptr); @@ -24,19 +23,19 @@ int PCU_Comm_Free2(PCUHandle* h) { return PCU_SUCCESS; } -int PCU_Comm_Self2(PCUHandle h) { +int PCU_Comm_Self(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Self called before Comm_Init"); return static_cast(h.ptr)->Self(); } -int PCU_Comm_Peers2(PCUHandle h) { +int PCU_Comm_Peers(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Peers called before Comm_Init"); return static_cast(h.ptr)->Peers(); } -void PCU_Comm_Begin2(PCUHandle h) { +void PCU_Comm_Begin(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Begin called before Comm_Init"); static_cast(h.ptr)->Begin(); @@ -48,7 +47,7 @@ void PCU_Comm_Begin2(PCUHandle h) { This function should be called after PCU_Comm_Start and before PCU_Comm_Send. */ -int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void *data, size_t size) { +int PCU_Comm_Pack(PCU_t h, int to_rank, const void *data, size_t size) { if (h.ptr == nullptr) reel_fail("Comm_Pack called before Comm_Init"); return static_cast(h.ptr)->Pack(to_rank, data, size); @@ -61,7 +60,7 @@ int PCU_Comm_Pack2(PCUHandle h, int to_rank, const void *data, size_t size) { All buffers from this thread are sent out and receiving may begin after this call. */ -int PCU_Comm_Send2(PCUHandle h) { +int PCU_Comm_Send(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Send called before Comm_Init"); return static_cast(h.ptr)->Send(); @@ -78,7 +77,7 @@ int PCU_Comm_Send2(PCUHandle h) { Users should unpack all data from this buffer before calling this function again, because the previously received buffer is destroyed by the call. */ -bool PCU_Comm_Listen2(PCUHandle h) { +bool PCU_Comm_Listen(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Listen called before Comm_Init"); return static_cast(h.ptr)->Listen(); @@ -87,13 +86,13 @@ bool PCU_Comm_Listen2(PCUHandle h) { /** \brief Returns in * \a from_rank the sender of the current received buffer. \details This function should be called after a successful PCU_Comm_Listen. */ -int PCU_Comm_Sender2(PCUHandle h) { +int PCU_Comm_Sender(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Sender called before Comm_Init"); return static_cast(h.ptr)->Sender(); } -bool PCU_Comm_Unpacked2(PCUHandle h) { +bool PCU_Comm_Unpacked(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Unpacked called before Comm_Init"); return static_cast(h.ptr)->Unpacked(); @@ -109,20 +108,20 @@ bool PCU_Comm_Unpacked2(PCUHandle h) { Users must ensure that there remain \a size bytes to be unpacked, PCU_Comm_Unpacked can help with this. */ -int PCU_Comm_Unpack2(PCUHandle h, void *data, size_t size) { +int PCU_Comm_Unpack(PCU_t h, void *data, size_t size) { if (h.ptr == nullptr) reel_fail("Comm_Unpack called before Comm_Init"); return static_cast(h.ptr)->Unpack(data, size); } -void PCU_Comm_Order2(PCUHandle h, bool on) { +void PCU_Comm_Order(PCU_t h, bool on) { if (h.ptr == nullptr) reel_fail("Comm_Order called before Comm_Init"); static_cast(h.ptr)->Order(on); } /** \brief Blocking barrier over all threads. */ -void PCU_Barrier2(PCUHandle h) { +void PCU_Barrier(PCU_t h) { if (h.ptr == nullptr) reel_fail("Barrier called before Comm_Init"); static_cast(h.ptr)->Barrier(); @@ -134,13 +133,13 @@ void PCU_Barrier2(PCUHandle h) { After this call, p[i] will contain the sum of all p[i]'s given by each rank. */ -void PCU_Add_Doubles2(PCUHandle h, double *p, size_t n) { +void PCU_Add_Doubles(PCU_t h, double *p, size_t n) { if (h.ptr == nullptr) reel_fail("Add_Doubles called before Comm_Init"); static_cast(h.ptr)->Add(p, n); } -double PCU_Add_Double2(PCUHandle h, double x) { +double PCU_Add_Double(PCU_t h, double x) { if (h.ptr == nullptr) reel_fail("Add_Double called before Comm_Init"); return static_cast(h.ptr)->Add(x); @@ -148,13 +147,13 @@ double PCU_Add_Double2(PCUHandle h, double x) { /** \brief Performs an Allreduce minimum of double arrays. */ -void PCU_Min_Doubles2(PCUHandle h, double *p, size_t n) { +void PCU_Min_Doubles(PCU_t h, double *p, size_t n) { if (h.ptr == nullptr) reel_fail("Min_Doubles called before Comm_Init"); static_cast(h.ptr)->Min(p, n); } -double PCU_Min_Double2(PCUHandle h, double x) { +double PCU_Min_Double(PCU_t h, double x) { if (h.ptr == nullptr) reel_fail("Min_Double called before Comm_Init"); return static_cast(h.ptr)->Min(x); @@ -162,13 +161,13 @@ double PCU_Min_Double2(PCUHandle h, double x) { /** \brief Performs an Allreduce maximum of double arrays. */ -void PCU_Max_Doubles2(PCUHandle h, double *p, size_t n) { +void PCU_Max_Doubles(PCU_t h, double *p, size_t n) { if (h.ptr == nullptr) reel_fail("Max_Doubles called before Comm_Init"); static_cast(h.ptr)->Max(p, n); } -double PCU_Max_Double2(PCUHandle h, double x) { +double PCU_Max_Double(PCU_t h, double x) { if (h.ptr == nullptr) reel_fail("Max_Double called before Comm_Init"); return static_cast(h.ptr)->Max(x); @@ -176,13 +175,13 @@ double PCU_Max_Double2(PCUHandle h, double x) { /** \brief Performs an Allreduce sum of integers */ -void PCU_Add_Ints2(PCUHandle h, int *p, size_t n) { +void PCU_Add_Ints(PCU_t h, int *p, size_t n) { if (h.ptr == nullptr) reel_fail("Add_Ints called before Comm_Init"); static_cast(h.ptr)->Add(p, n); } -int PCU_Add_Int2(PCUHandle h, int x) { +int PCU_Add_Int(PCU_t h, int x) { if (h.ptr == nullptr) reel_fail("Add_Int called before Comm_Init"); return static_cast(h.ptr)->Add(x); @@ -190,13 +189,13 @@ int PCU_Add_Int2(PCUHandle h, int x) { /** \brief Performs an Allreduce sum of long integers */ -void PCU_Add_Longs2(PCUHandle h, long *p, size_t n) { +void PCU_Add_Longs(PCU_t h, long *p, size_t n) { if (h.ptr == nullptr) reel_fail("Add_Longs called before Comm_Init"); static_cast(h.ptr)->Add(p, n); } -long PCU_Add_Long2(PCUHandle h, long x) { +long PCU_Add_Long(PCU_t h, long x) { if (h.ptr == nullptr) reel_fail("Add_Long called before Comm_Init"); return static_cast(h.ptr)->Add(x); @@ -204,13 +203,13 @@ long PCU_Add_Long2(PCUHandle h, long x) { /** \brief Performs an Allreduce sum of size_t unsigned integers */ -void PCU_Add_SizeTs2(PCUHandle h, size_t *p, size_t n) { +void PCU_Add_SizeTs(PCU_t h, size_t *p, size_t n) { if (h.ptr == nullptr) reel_fail("Add_SizeTs called before Comm_Init"); static_cast(h.ptr)->Add(p, n); } -size_t PCU_Add_SizeT2(PCUHandle h, size_t x) { +size_t PCU_Add_SizeT(PCU_t h, size_t x) { if (h.ptr == nullptr) reel_fail("Add_SizeT called before Comm_Init"); return static_cast(h.ptr)->Add(x); @@ -218,13 +217,13 @@ size_t PCU_Add_SizeT2(PCUHandle h, size_t x) { /** \brief Performs an Allreduce minimum of size_t unsigned integers */ -void PCU_Min_SizeTs2(PCUHandle h, size_t *p, size_t n) { +void PCU_Min_SizeTs(PCU_t h, size_t *p, size_t n) { if (h.ptr == nullptr) reel_fail("Min_SizeTs called before Comm_Init"); static_cast(h.ptr)->Min(p, n); } -size_t PCU_Min_SizeT2(PCUHandle h, size_t x) { +size_t PCU_Min_SizeT(PCU_t h, size_t x) { if (h.ptr == nullptr) reel_fail("Min_SizeT called before Comm_Init"); return static_cast(h.ptr)->Min(x); @@ -232,13 +231,13 @@ size_t PCU_Min_SizeT2(PCUHandle h, size_t x) { /** \brief Performs an Allreduce maximum of size_t unsigned integers */ -void PCU_Max_SizeTs2(PCUHandle h, size_t *p, size_t n) { +void PCU_Max_SizeTs(PCU_t h, size_t *p, size_t n) { if (h.ptr == nullptr) reel_fail("Max_SizeTs called before Comm_Init"); static_cast(h.ptr)->Max(p, n); } -size_t PCU_Max_SizeT2(PCUHandle h, size_t x) { +size_t PCU_Max_SizeT(PCU_t h, size_t x) { if (h.ptr == nullptr) reel_fail("Max_SizeT called before Comm_Init"); return static_cast(h.ptr)->Max(x); @@ -250,26 +249,26 @@ size_t PCU_Max_SizeT2(PCUHandle h, size_t x) { After this call, p[i] will contain the sum of all p[i]'s given by ranks lower than the calling rank. */ -void PCU_Exscan_Ints2(PCUHandle h, int *p, size_t n) { +void PCU_Exscan_Ints(PCU_t h, int *p, size_t n) { if (h.ptr == nullptr) reel_fail("Exscan_Ints called before Comm_Init"); static_cast(h.ptr)->Exscan(p, n); } -int PCU_Exscan_Int2(PCUHandle h, int x) { +int PCU_Exscan_Int(PCU_t h, int x) { if (h.ptr == nullptr) reel_fail("Exscan_Int called before Comm_Init"); return static_cast(h.ptr)->Exscan(x); } /** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs2(PCUHandle h, long *p, size_t n) { +void PCU_Exscan_Longs(PCU_t h, long *p, size_t n) { if (h.ptr == nullptr) reel_fail("Exscan_Longs called before Comm_Init"); static_cast(h.ptr)->Exscan(p, n); } -long PCU_Exscan_Long2(PCUHandle h, long x) { +long PCU_Exscan_Long(PCU_t h, long x) { if (h.ptr == nullptr) reel_fail("Exscan_Long called before Comm_Init"); return static_cast(h.ptr)->Exscan(x); @@ -277,13 +276,13 @@ long PCU_Exscan_Long2(PCUHandle h, long x) { /** \brief Performs an Allreduce minimum of int arrays. */ -void PCU_Min_Ints2(PCUHandle h, int *p, size_t n) { +void PCU_Min_Ints(PCU_t h, int *p, size_t n) { if (h.ptr == nullptr) reel_fail("Min_Ints called before Comm_Init"); static_cast(h.ptr)->Min(p, n); } -int PCU_Min_Int2(PCUHandle h, int x) { +int PCU_Min_Int(PCU_t h, int x) { if (h.ptr == nullptr) reel_fail("Min_Int called before Comm_Init"); return static_cast(h.ptr)->Min(x); @@ -291,26 +290,26 @@ int PCU_Min_Int2(PCUHandle h, int x) { /** \brief Performs an Allreduce maximum of int arrays. */ -void PCU_Max_Ints2(PCUHandle h, int *p, size_t n) { +void PCU_Max_Ints(PCU_t h, int *p, size_t n) { if (h.ptr == nullptr) reel_fail("Max_Ints called before Comm_Init"); static_cast(h.ptr)->Max(p, n); } -int PCU_Max_Int2(PCUHandle h, int x) { +int PCU_Max_Int(PCU_t h, int x) { if (h.ptr == nullptr) reel_fail("Max_Int called before Comm_Init"); return static_cast(h.ptr)->Max(x); } /** \brief Performs an Allreduce maximum of long arrays. */ -void PCU_Max_Longs2(PCUHandle h, long *p, size_t n) { +void PCU_Max_Longs(PCU_t h, long *p, size_t n) { if (h.ptr == nullptr) reel_fail("Max_Longs called before Comm_Init"); static_cast(h.ptr)->Max(p, n); } -long PCU_Max_Long2(PCUHandle h, long x) { +long PCU_Max_Long(PCU_t h, long x) { if (h.ptr == nullptr) reel_fail("Max_Long called before Comm_Init"); return static_cast(h.ptr)->Max(x); @@ -318,7 +317,7 @@ long PCU_Max_Long2(PCUHandle h, long x) { /** \brief Performs a parallel logical OR reduction */ -int PCU_Or2(PCUHandle h, int c) { +int PCU_Or(PCU_t h, int c) { if (h.ptr == nullptr) reel_fail("Or called before Comm_Init"); return static_cast(h.ptr)->Or(c); @@ -326,7 +325,7 @@ int PCU_Or2(PCUHandle h, int c) { /** \brief Performs a parallel logical AND reduction */ -int PCU_And2(PCUHandle h, int c) { +int PCU_And(PCU_t h, int c) { if (h.ptr == nullptr) reel_fail("And called before Comm_Init"); return static_cast(h.ptr)->And(c); @@ -334,7 +333,7 @@ int PCU_And2(PCUHandle h, int c) { /** \brief Returns the unique rank of the calling process. */ -int PCU_Proc_Self2(PCUHandle h) { +int PCU_Proc_Self(PCU_t h) { if (h.ptr == nullptr) reel_fail("Proc_Self called before Comm_Init"); return static_cast(h.ptr)->Self(); @@ -342,7 +341,7 @@ int PCU_Proc_Self2(PCUHandle h) { /** \brief Returns the number of processes. */ -int PCU_Proc_Peers2(PCUHandle h) { +int PCU_Proc_Peers(PCU_t h) { if (h.ptr == nullptr) reel_fail("Proc_Peers called before Comm_Init"); return static_cast(h.ptr)->Peers(); @@ -350,7 +349,7 @@ int PCU_Proc_Peers2(PCUHandle h) { /** \brief Similar to PCU_Comm_Self, returns the rank as an argument. */ -int PCU_Comm_Rank2(PCUHandle h, int *rank) { +int PCU_Comm_Rank(PCU_t h, int *rank) { if (h.ptr == nullptr) reel_fail("Comm_Rank called before Comm_Init"); *rank = static_cast(h.ptr)->Self(); @@ -358,7 +357,7 @@ int PCU_Comm_Rank2(PCUHandle h, int *rank) { } /** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size2(PCUHandle h, int *size) { +int PCU_Comm_Size(PCU_t h, int *size) { if (h.ptr == nullptr) reel_fail("Comm_Size called before Comm_Init"); *size = static_cast(h.ptr)->Peers(); @@ -366,7 +365,7 @@ int PCU_Comm_Size2(PCUHandle h, int *size) { } /** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized2(PCUHandle h) { return h.ptr != nullptr; } +bool PCU_Comm_Initialized(PCU_t h) { return h.ptr != nullptr; } @@ -375,7 +374,7 @@ bool PCU_Comm_Initialized2(PCUHandle h) { return h.ptr != nullptr; } This function should be called after PCU_Comm_Start and before PCU_Comm_Send. */ -int PCU_Comm_Packed2(PCUHandle h, int to_rank, size_t *size) { +int PCU_Comm_Packed(PCU_t h, int to_rank, size_t *size) { if (h.ptr == nullptr) reel_fail("Comm_Packed called before Comm_Init"); return static_cast(h.ptr)->Packed(to_rank, size); @@ -390,14 +389,14 @@ int PCU_Comm_Packed2(PCUHandle h, int to_rank, size_t *size) { PCU_Comm_Send. If this function is used, PCU_Comm_Pack should not be used. */ -int PCU_Comm_Write2(PCUHandle h, int to_rank, const void *data, size_t size) { +int PCU_Comm_Write(PCU_t h, int to_rank, const void *data, size_t size) { if (h.ptr == nullptr) reel_fail("Comm_Write called before Comm_Init"); return static_cast(h.ptr)->Write(to_rank, data, size); } /** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive2(PCUHandle h) { +bool PCU_Comm_Receive(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Receive called before Comm_Init"); return static_cast(h.ptr)->Receive(); @@ -415,20 +414,20 @@ bool PCU_Comm_Receive2(PCUHandle h) { Note that the address * \a data points into a PCU buffer, so it is strongly recommended that this data be read and not modified. */ -bool PCU_Comm_Read2(PCUHandle h, int *from_rank, void **data, size_t *size) { +bool PCU_Comm_Read(PCU_t h, int *from_rank, void **data, size_t *size) { if (h.ptr == nullptr) reel_fail("Comm_Read called before Comm_Init"); return static_cast(h.ptr)->Read(from_rank, data, size); } -void PCU_Debug_Open2(PCUHandle h) { +void PCU_Debug_Open(PCU_t h) { if (h.ptr == nullptr) reel_fail("Debug_Open called before Comm_Init"); static_cast(h.ptr)->DebugOpen(); } /** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print2(PCUHandle h, const char *format, ...) { +void PCU_Debug_Print(PCU_t h, const char *format, ...) { if (h.ptr == nullptr) reel_fail("Debug_Print called before Comm_Init"); va_list arglist; @@ -438,7 +437,7 @@ void PCU_Debug_Print2(PCUHandle h, const char *format, ...) { } /** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From2(PCUHandle h, int *from_rank) { +int PCU_Comm_From(PCU_t h, int *from_rank) { if (h.ptr == nullptr) reel_fail("Comm_From called before Comm_Init"); return static_cast(h.ptr)->From(from_rank); @@ -449,7 +448,7 @@ int PCU_Comm_From2(PCUHandle h, int *from_rank) { The size returned will be the total received size regardless of how much unpacking has been done. */ -int PCU_Comm_Received2(PCUHandle h, size_t *size) { +int PCU_Comm_Received(PCU_t h, size_t *size) { if (h.ptr == nullptr) reel_fail("Comm_Received called before Comm_Init"); return static_cast(h.ptr)->Received(size); @@ -461,7 +460,7 @@ int PCU_Comm_Received2(PCUHandle h, size_t *size) { and an internal pointer to that data is returned. The returned pointer must not be freed by the user. */ -void *PCU_Comm_Extract2(PCUHandle h, size_t size) { +void *PCU_Comm_Extract(PCU_t h, size_t size) { if (h.ptr == nullptr) reel_fail("Comm_Extract called before Comm_Init"); return static_cast(h.ptr)->Extract(size); @@ -475,7 +474,7 @@ void *PCU_Comm_Extract2(PCUHandle h, size_t size) { in the previous communicator. This is a very heavy weight function and should be used sparingly. */ -void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm) { +void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm) { if (h.ptr == nullptr) reel_fail("Switch_Comm called before Comm_Init"); static_cast(h.ptr)->SwitchMPIComm(new_comm); @@ -486,7 +485,7 @@ void PCU_Switch_Comm2(PCUHandle h, MPI_Comm new_comm) { most recent PCU_Switch_Comm call, or MPI_COMM_WORLD otherwise. */ -MPI_Comm PCU_Get_Comm2(PCUHandle h) { +MPI_Comm PCU_Get_Comm(PCU_t h) { if (h.ptr == nullptr) reel_fail("Get_Comm called before Comm_Init"); return static_cast(h.ptr)->GetMPIComm(); @@ -494,17 +493,11 @@ MPI_Comm PCU_Get_Comm2(PCUHandle h) { /** \brief Return the time in seconds since some time in the past */ -double PCU_Time2(void) { return pcu::Time(); } - -void PCU_Protect2(void) { return pcu::Protect(); } - -double PCU_GetMem2(void) { return pcu::GetMem(); } - - -/** \brief Return the global PCU - */ +double PCU_Time(void) { return pcu::Time(); } +void PCU_Protect(void) { return pcu::Protect(); } +double PCU_GetMem(void) { return pcu::GetMem(); } } \ No newline at end of file diff --git a/pcu/pcu_defines.h b/pcu/pcu_defines.h index 22957adab..acf8861e5 100644 --- a/pcu/pcu_defines.h +++ b/pcu/pcu_defines.h @@ -14,7 +14,7 @@ extern "C"{ #endif -struct PCUHandle { +struct PCU_t { void* ptr; }; diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index 93ad2345c..ec6eb21f4 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,7 +8,7 @@ *******************************************************************************/ #include "pcu_io.h" -#include "PCU2.h" +#include "PCU.h" #include "noto_malloc.h" #include "pcu_buffer.h" #include "pcu_util.h" @@ -158,18 +158,18 @@ static void close_compressed(pcu_file* pf) * Ideally, the filesystem handles the load and this can be * removed. */ -FILE* pcu_group_open(PCUHandle h, const char* path, bool write) { +FILE* pcu_group_open(PCU_t h, const char* path, bool write) { FILE* fp = NULL; - const int rank = PCU_Comm_Self2(h); + const int rank = PCU_Comm_Self(h); const char* mode = write ? "w" : "r"; const int group_size = 4096; - const int q = PCU_Comm_Peers2(h)/group_size; - const int r = PCU_Comm_Peers2(h)%group_size; + const int q = PCU_Comm_Peers(h)/group_size; + const int r = PCU_Comm_Peers(h)%group_size; const int groups = q + ( r > 0 ); if(!rank && groups > 1) { fprintf(stderr, "pcu peers %d max group size %d posix groups %d\n", - PCU_Comm_Peers2(h), group_size, groups); + PCU_Comm_Peers(h), group_size, groups); } for(int i=0; icompress = compress; @@ -355,14 +355,14 @@ void pcu_write_string (pcu_file * f, const char * p) pcu_write (f, p, len + 1); } -FILE* pcu_open_parallel(PCUHandle h, const char* prefix, const char* ext) +FILE* pcu_open_parallel(PCU_t h, const char* prefix, const char* ext) { //max_rank_chars = strlen("4294967296"), 4294967296 = 2^32 ~= INT_MAX static const size_t max_rank_chars = 10; size_t path_size = strlen(prefix) + max_rank_chars + strlen(ext) + 1; char* path = noto_malloc(path_size); int rank; - PCU_Comm_Rank2(h, &rank); + PCU_Comm_Rank(h, &rank); snprintf(path,path_size,"%s%d.%s",prefix,rank,ext); FILE* file = fopen(path, "w"); noto_free(path); diff --git a/pcu/pcu_io.h b/pcu/pcu_io.h index 88edece46..ee223b082 100644 --- a/pcu/pcu_io.h +++ b/pcu/pcu_io.h @@ -20,9 +20,9 @@ extern "C" { #endif struct pcu_file; -typedef struct PCUHandle PCUHandle; +typedef struct PCU_t PCU_t; -struct pcu_file* pcu_fopen(PCUHandle h, const char* path, bool write, bool compress); +struct pcu_file* pcu_fopen(PCU_t h, const char* path, bool write, bool compress); void pcu_fclose (struct pcu_file * pf); void pcu_read(struct pcu_file* f, char* p, size_t n); void pcu_write(struct pcu_file* f, const char* p, size_t n); @@ -35,8 +35,8 @@ void pcu_write_doubles(struct pcu_file* f, double* p, size_t n); void pcu_read_string(struct pcu_file* f, char** p); void pcu_write_string(struct pcu_file* f, const char* p); -FILE* pcu_open_parallel(PCUHandle h, const char* prefix, const char* ext); -FILE* pcu_group_open(PCUHandle h, const char* path, bool write); +FILE* pcu_open_parallel(PCU_t h, const char* prefix, const char* ext); +FILE* pcu_group_open(PCU_t h, const char* path, bool write); void pcu_swap_doubles(double* p, size_t n); void pcu_swap_unsigneds(unsigned* p, size_t n); diff --git a/phasta/phCook.cc b/phasta/phCook.cc index df973f6dc..07a5ebb05 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -176,7 +176,7 @@ namespace ph { } void preprocess(apf::Mesh2* m, Input& in, Output& out, BCs& bcs) { - PCUHandle h; + PCU_t h; h.ptr = static_cast(m->getPCU()); phastaio_initStats(h); if(m->getPCU()->Peers() > 1) diff --git a/phasta/phIO.c b/phasta/phIO.c index 50b5a4845..d94caf120 100644 --- a/phasta/phIO.c +++ b/phasta/phIO.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -70,7 +70,7 @@ static void parse_header(char* header, char** name, long* bytes, } } -static int find_header(PCUHandle h, FILE* f, const char* name, char* found, char header[PH_LINE]) +static int find_header(PCU_t h, FILE* f, const char* name, char* found, char header[PH_LINE]) { char* hname; long bytes; @@ -88,7 +88,7 @@ static int find_header(PCUHandle h, FILE* f, const char* name, char* found, char } fseek(f, bytes, SEEK_CUR); } - if (!PCU_Comm_Self2(h) && strlen(name) > 0) + if (!PCU_Comm_Self(h) && strlen(name) > 0) lion_eprint(1,"warning: phIO could not find \"%s\"\n",name); return 0; } @@ -102,7 +102,7 @@ static void write_magic_number(FILE* f) fprintf(f,"\n"); } -static int seek_after_header(PCUHandle h, FILE* f, const char* name) +static int seek_after_header(PCU_t h, FILE* f, const char* name) { char dummy[PH_LINE]; char found[PH_LINE]; @@ -115,11 +115,11 @@ static void my_fread(void* p, size_t size, size_t nmemb, FILE* f) PCU_ALWAYS_ASSERT(r == nmemb); } -static int read_magic_number(PCUHandle h, FILE* f) +static int read_magic_number(PCU_t h, FILE* f) { int magic; if (!seek_after_header(h, f, magic_name)) { - if (!PCU_Comm_Self2(h)) + if (!PCU_Comm_Self(h)) lion_eprint(1,"warning: not swapping bytes\n"); rewind(f); return 0; @@ -162,11 +162,11 @@ static void parse_params(char* header, long* bytes, *step = params[STEP_PARAM]; } -int ph_should_swap(FILE* f, PCUHandle h) { +int ph_should_swap(FILE* f, PCU_t h) { return read_magic_number(h, f); } -int ph_read_field(FILE* f, PCUHandle h, const char* field, int swap, +int ph_read_field(FILE* f, PCU_t h, const char* field, int swap, double** data, int* nodes, int* vars, int* step, char* hname) { long bytes, n; diff --git a/phasta/phIO.h b/phasta/phIO.h index 921a2c0c1..4a53ead2f 100644 --- a/phasta/phIO.h +++ b/phasta/phIO.h @@ -18,14 +18,14 @@ void ph_write_ints(FILE* f, const char* name, int* data, * swapped to account for endianness * @return 1 if swapping is required, 0 otherwise */ -int ph_should_swap(FILE* f, PCUHandle h); +int ph_should_swap(FILE* f, PCU_t h); /** * @brief read a field * @return 1 if no data block was read, 2 if data block read, 0 otherwise */ -int ph_read_field(FILE* f, PCUHandle h, const char* field, int swap, +int ph_read_field(FILE* f, PCU_t h, const char* field, int swap, double** data, int* nodes, int* vars, int* step, char* hname); void ph_write_field(FILE* f, const char* field, double* data, int nodes, int vars, int step); diff --git a/phasta/phRestart.cc b/phasta/phRestart.cc index 3410a8b47..c5fec0650 100644 --- a/phasta/phRestart.cc +++ b/phasta/phRestart.cc @@ -466,7 +466,7 @@ static std::string buildRestartFileName(std::string prefix, int step, pcu::PCU * } void readAndAttachFields(Input& in, apf::Mesh* m) { - PCUHandle h; + PCU_t h; h.ptr = static_cast(m->getPCU()); phastaio_initStats(h); double t0 = pcu::Time(); diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index c5af90606..965fa0f3d 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -33,7 +33,7 @@ void phastaio_time(phastaioTime* t) { *t = _rdtsc(); //intel intrinsic } /* determine the reference clock frequency */ -static size_t phastaio_getCyclesPerMicroSec(PCUHandle h) { +static size_t phastaio_getCyclesPerMicroSec(PCU_t h) { const size_t usec = 5*MILLION; size_t cpus, cycles; phastaioTime t0, t1; @@ -45,7 +45,7 @@ static size_t phastaio_getCyclesPerMicroSec(PCUHandle h) { phastaio_time(&t1); cycles = t1 - t0; cpus = ((double)cycles)/(usec); - if(!PCU_Comm_Self2(h)) { + if(!PCU_Comm_Self(h)) { std::stringstream ss; ss << "cycles " << cycles << " us " << usec << " cycles per micro second " << cpus << "\n"; @@ -141,12 +141,12 @@ static const char* getFileName() { return names[phastaio_global_stats.fileIdx]; } -static void printMinMaxAvgSzt(const char* key, size_t v, PCUHandle h) { - size_t min = PCU_Min_SizeT2(h, v); - size_t max = PCU_Max_SizeT2(h, v); - size_t tot = PCU_Add_SizeT2(h, v); - double avg = ((double)tot)/PCU_Comm_Peers2(h); - if(!PCU_Comm_Self2(h)) { +static void printMinMaxAvgSzt(const char* key, size_t v, PCU_t h) { + size_t min = PCU_Min_SizeT(h, v); + size_t max = PCU_Max_SizeT(h, v); + size_t tot = PCU_Add_SizeT(h, v); + double avg = ((double)tot)/PCU_Comm_Peers(h); + if(!PCU_Comm_Self(h)) { std::stringstream ss; ss << getFileName() << "_" << key << "min max avg" << min << " " << max << " " << avg << "\n"; @@ -155,12 +155,12 @@ static void printMinMaxAvgSzt(const char* key, size_t v, PCUHandle h) { } } -static void printMinMaxAvgDbl(const char* key, double v, PCUHandle h) { - double min = PCU_Min_Double2(h, v); - double max = PCU_Max_Double2(h, v); - double tot = PCU_Add_Double2(h, v); - double avg = tot/PCU_Comm_Peers2(h); - if(!PCU_Comm_Self2(h)) +static void printMinMaxAvgDbl(const char* key, double v, PCU_t h) { + double min = PCU_Min_Double(h, v); + double max = PCU_Max_Double(h, v); + double tot = PCU_Add_Double(h, v); + double avg = tot/PCU_Comm_Peers(h); + if(!PCU_Comm_Self(h)) lion_eprint(1, "%s_%s min max avg %f %f %f\n", getFileName(), key, min, max, avg); } @@ -215,8 +215,8 @@ static size_t phastaio_getCloseTime() { return phastaio_global_stats.closeTime[i]; } -void phastaio_printStats(PCUHandle h) { - if(!PCU_Comm_Self2(h)) { +void phastaio_printStats(PCU_t h) { + if(!PCU_Comm_Self(h)) { const size_t us = 1000; phastaioTime t0,t1; size_t elapsed; @@ -233,9 +233,9 @@ void phastaio_printStats(PCUHandle h) { size_t totalus = 0; size_t totalbytes = 0; phastaio_setfile(chefFile); - if(!PCU_Comm_Self2(h)) + if(!PCU_Comm_Self(h)) lion_eprint(1, "phastaio_filename %s\n", getFileName()); - int reads = PCU_Max_Int2(h, (int)phastaio_getReads()); + int reads = PCU_Max_Int(h, (int)phastaio_getReads()); if(reads) { totalus += phastaio_getReadTime(); totalbytes += phastaio_getReadBytes(); @@ -249,7 +249,7 @@ void phastaio_printStats(PCUHandle h) { * us 1s 10^6B s */ } - int writes = PCU_Max_Int2(h, (int)phastaio_getWrites()); + int writes = PCU_Max_Int(h, (int)phastaio_getWrites()); if(writes) { totalus += phastaio_getWriteTime(); totalbytes += phastaio_getWriteBytes(); @@ -259,13 +259,13 @@ void phastaio_printStats(PCUHandle h) { printMinMaxAvgDbl("writeBandwidth (MB/s)", ((double)phastaio_getWriteBytes())/phastaio_getWriteTime(), h); } - int opens = PCU_Max_Int2(h, (int)phastaio_getOpens()); + int opens = PCU_Max_Int(h, (int)phastaio_getOpens()); if(opens) { totalus += phastaio_getOpenTime(); printMinMaxAvgSzt("opens", phastaio_getOpens(), h); printMinMaxAvgSzt("openTime (us)", phastaio_getOpenTime(), h); } - int closes = PCU_Max_Int2(h, (int)phastaio_getCloses()); + int closes = PCU_Max_Int(h, (int)phastaio_getCloses()); if(closes) { totalus += phastaio_getCloseTime(); printMinMaxAvgSzt("closes", phastaio_getCloses(), h); @@ -280,7 +280,7 @@ void phastaio_printStats(PCUHandle h) { } } -void phastaio_initStats(PCUHandle h) { +void phastaio_initStats(PCU_t h) { (void) h; #ifdef __INTEL_COMPILER phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(h); diff --git a/phasta/phiotimer.h b/phasta/phiotimer.h index 90ebec4e8..ec1dc3dcd 100644 --- a/phasta/phiotimer.h +++ b/phasta/phiotimer.h @@ -5,13 +5,12 @@ \brief timers for reading and writing phasta files */ -//#include #ifdef __cplusplus extern "C" { #endif -typedef struct PCUHandle PCUHandle; +typedef struct PCU_t PCU_t; #define PHASTAIO_READTIME(cmd,bytes) {\ phastaioTime t0,t1;\ @@ -85,9 +84,9 @@ void phastaio_addOpenTime(size_t t); /* \brief accumulate time closing */ void phastaio_addCloseTime(size_t t); /* \brief initialize the counters and timers */ -void phastaio_initStats(PCUHandle h); +void phastaio_initStats(PCU_t h); /* \brief print io information */ -void phastaio_printStats(PCUHandle h); +void phastaio_printStats(PCU_t h); /* \brief set the current file to record counters and timers for * \detail see phastaio_file enum */ void phastaio_setfile(int f); diff --git a/test/cgns.cc b/test/cgns.cc index b5ac692af..d92c69a9e 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include // @@ -178,7 +178,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: gmi_register_null(); gmi_register_mesh(); gmi_model *g = gmi_load(".null"); - PCUHandle h; + PCU_t h; h.ptr = static_cast(PCUObj); apf::Mesh2 *m = apf::loadMdsFromCGNS2(h, g, argv1.c_str(), cgnsBCMap, meshData); m->verify(); From deba0526ae687f3c47ebb6671d4fc381fef64551 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 9 Jul 2024 12:40:40 -0400 Subject: [PATCH 119/141] pumi_sys functions using pumi::instance to access PCU, added setPCU and getPCU functions to pumi::instance() --- pumi/pumi.h | 17 ++++---- pumi/pumi_field.cc | 24 +++++------ pumi/pumi_geom.cc | 8 ++-- pumi/pumi_ghost.cc | 26 ++++++------ pumi/pumi_mentity.cc | 18 ++++---- pumi/pumi_mesh.cc | 84 +++++--------------------------------- pumi/pumi_numbering.cc | 12 +++--- pumi/pumi_sys.cc | 24 +++++------ test/cgns.cc | 1 + test/constructThenGhost.cc | 3 +- test/pumi.cc | 43 +++++++++---------- test/pumiLoadMesh.cc | 1 + test/xgc_split.cc | 8 ++-- 13 files changed, 105 insertions(+), 164 deletions(-) diff --git a/pumi/pumi.h b/pumi/pumi.h index bfdf4e346..b4e58494d 100644 --- a/pumi/pumi.h +++ b/pumi/pumi.h @@ -90,6 +90,8 @@ class pumi static pumi _instance; return &_instance; }; + void setPCU(pcu::PCU *newPCU) {PCUObj = newPCU;} + [[nodiscard]] pcu::PCU* getPCU() const noexcept {return PCUObj;} pMesh mesh; pGeom model; @@ -100,6 +102,8 @@ class pumi pMeshTag ghost_tag; std::vector ghost_vec[4]; std::vector ghosted_vec[4]; + private: + pcu::PCU *PCUObj; }; //************************************ @@ -107,16 +111,15 @@ class pumi // 0- SYSTEM-LEVEL FUNCTIONS //************************************ //************************************ -void pumi_finalize(pcu::PCU *PCUObj, bool do_mpi_finalize=true); -int pumi_size(pcu::PCU *PCUObj); -int pumi_rank(pcu::PCU *PCUObj); +int pumi_size(); +int pumi_rank(); -void pumi_sync(pcu::PCU *PCUObj); -void pumi_printSys(pcu::PCU *PCUObj); +void pumi_sync(); +void pumi_printSys(); double pumi_getTime(); double pumi_getMem(); -void pumi_printTimeMem(const char* msg, double time, double memory, pcu::PCU *PCUObj); +void pumi_printTimeMem(const char* msg, double time, double memory); //************************************ //************************************ @@ -139,7 +142,7 @@ void pumi_geom_createID(pGeom g); // generate sequential ID starting from 1 int pumi_geom_getNumEnt(pGeom g, int d); pGeomEnt pumi_geom_findEnt(pGeom g, int d, int id); -void pumi_geom_print(pGeom g, pcu::PCU *PCUObj, bool print_ent=false); +void pumi_geom_print(pGeom g, bool print_ent=false); // Geometric Entity // get geometric entity's dimension diff --git a/pumi/pumi_field.cc b/pumi/pumi_field.cc index 74c520126..61e0ed14d 100644 --- a/pumi/pumi_field.cc +++ b/pumi/pumi_field.cc @@ -205,7 +205,7 @@ void pumi_field_multiply(pField f1, double d, pField f2) void pumi_field_print(pField f) //******************************************************* { - pumi_sync(getMesh(f)->getPCU()); + pumi_sync(); apf::Mesh* m = getMesh(f); if (!m->findTag("global_id")) pumi_mesh_createGlobalID((pMesh)m); @@ -228,20 +228,20 @@ void pumi_field_print(pField f) switch (n) { case 1: { - std::cout<<"[p"<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())<<"] field "<getPCU())) std::cout<<__func__<<" failed for field " + default: if (!pumi_rank()) std::cout<<__func__<<" failed for field " <getPCU())) // master + if (!pumi_rank()) // master { lion_oprint(1," - verifying fields: "); for (size_t nf = 0; nf < fields.size(); ++nf) diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index 1c19467b5..ef2f1239f 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -46,7 +46,7 @@ pGeom pumi_geom_load(const char* filename, pcu::PCU *PCUObj, const char* model_t else if (!strcmp(model_type,"analytic")) return pumi_geom_load(gmi_make_analytic(), PCUObj, model_type, filename, geom_load_fp); else - if (!pumi_rank(PCUObj)) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); + if (!pumi_rank()) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); return NULL; } @@ -73,7 +73,7 @@ pGeom pumi_geom_load(gmi_model* gm, pcu::PCU *PCUObj, const char* model_type, } else { - if (!pumi_rank(PCUObj)) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); + if (!pumi_rank()) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); return NULL; } @@ -191,9 +191,9 @@ void pumi_giter_reset(gIter iter) iter->reset(); } -void pumi_geom_print (pGeom g, pcu::PCU *PCUObj, bool print_ent) +void pumi_geom_print (pGeom g, bool print_ent) { - if (PCUObj->Self()) return; + if (pumi::instance()->getPCU()->Self()) return; std::cout<<"\n=== model entity and tag info === \n"; std::cout<<"# global geom ent: v "<size(0)<<", e " <size(1)<<", f "<size(2)<<", r "<size(3)<<"\n"; diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc index d164ff79c..0a6235b76 100644 --- a/pumi/pumi_ghost.cc +++ b/pumi/pumi_ghost.cc @@ -458,7 +458,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int layer=2; layergetPCU()); ++pid) + for (int pid=0; pid::iterator off_it=off_bridge_set[layer][pid].begin(); off_it!=off_bridge_set[layer][pid].end(); ++off_it) @@ -484,7 +484,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, } // clean up for (int i=0; igetPCU());++j) + for (int j=0; jgetPCU()->Send(); @@ -576,7 +576,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, // the below runs only if num_layer>2 int global_num_off_part, local_num_off_part=0; for (int i=0; igetPCU());++j) + for (int j=0; jgetPCU()->GetMPIComm()); @@ -586,7 +586,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int layer=0; layergetPCU()); ++pid) + for (int pid=0; pid::iterator off_it=off_bridge_set[layer][pid].begin(); off_it!=off_bridge_set[layer][pid].end(); ++off_it) @@ -612,7 +612,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, } // clean up for (int i=0; igetPCU());++j) + for (int j=0; jgetPCU()->Send(); @@ -626,7 +626,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, r_layer=r_int[0]; r_pid=r_int[1]; - PCU_ALWAYS_ASSERT(r_layer<=num_layer && r_pidgetPCU())); + PCU_ALWAYS_ASSERT(r_layer<=num_layer && r_pidgetPCU());++j) + for (int j=0; jgetPCU()->GetMPIComm()); } // while global_off_part_brg @@ -710,7 +710,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, { if (m->getPCU()->Peers()==1 || num_layer==0) return; - int dummy=1, mesh_dim=m->getDimension(), self = pumi_rank(m->getPCU());; + int dummy=1, mesh_dim=m->getDimension(), self = pumi_rank();; // brid/ghost dim check if (brg_dim>=ghost_dim || 0>brg_dim || brg_dim>=mesh_dim || @@ -738,7 +738,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, std::set** off_bridge_set=new std::set*[num_layer+1]; for (int i=0; i[pumi_size(m->getPCU())]; + off_bridge_set[i]=new std::set[pumi_size()]; apf::MeshIterator* it = m->begin(brg_dim); while ((brg_ent = m->iterate(it))) @@ -821,7 +821,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, if (num_layer>=2) { for (int i=0; igetPCU());++j) + for (int j=0; jgetPCU()->GetMPIComm()); } @@ -832,7 +832,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, // clean up m->destroyTag(tag); for (int i=0; igetPCU());++j) + for (int j=0; j&) +void pumi_ghost_getInfo (pMesh, std::vector&) // ********************************************************* { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: not supported\n"; } diff --git a/pumi/pumi_mentity.cc b/pumi/pumi_mentity.cc index b029273d8..341a680ad 100644 --- a/pumi/pumi_mentity.cc +++ b/pumi/pumi_mentity.cc @@ -34,7 +34,7 @@ int pumi_ment_getNumAdj(pMeshEnt e, int target_dim) int ent_dim= apf::getDimension(pumi::instance()->mesh, e); if (ent_dim==target_dim) { - if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<": invalid target dimension "<mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<": invalid bridge/target dimension \n"; + if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<": invalid bridge/target dimension \n"; return; // error } @@ -178,7 +178,7 @@ int pumi_ment_getOwnPID(pMeshEnt e, pOwnership o) pMeshEnt pumi_ment_getOwnEnt(pMeshEnt e, pOwnership o) { int own_partid = pumi_ment_getOwnPID(e, o); - if (own_partid==pumi_rank(pumi::instance()->mesh->getPCU())) return e; + if (own_partid==pumi_rank()) return e; if (pumi::instance()->mesh->isShared(e)) { @@ -195,14 +195,14 @@ pMeshEnt pumi_ment_getOwnEnt(pMeshEnt e, pOwnership o) bool pumi_ment_isOwned(pMeshEnt e, pOwnership o) { if (!o) - return (pumi_ment_getOwnPID(e)==pumi_rank(pumi::instance()->mesh->getPCU())); + return (pumi_ment_getOwnPID(e)==pumi_rank()); return o->isOwned(e); } bool pumi_ment_isOn(pMeshEnt e, int partID) { // FIXME: include ghost copy - if (partID==pumi_rank(pumi::instance()->mesh->getPCU())) return true; + if (partID==pumi_rank()) return true; apf::Copies remotes; pumi::instance()->mesh->getRemotes(e,remotes); APF_ITERATE(Copies,remotes,rit) @@ -250,22 +250,22 @@ pMeshEnt pumi_ment_getRmt(pMeshEnt& e, int pid) void pumi_ment_setRmt(pMeshEnt, int, pMeshEnt) { - if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } void pumi_ment_deleteRmt (pMeshEnt, int) { - if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } void pumi_ment_cleanRmt (pMeshEnt) { - if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } void pumi_ment_setPtnTopology (pMeshEnt) { - if (!pumi_rank(pumi::instance()->mesh->getPCU())) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; + if (!pumi_rank()) std::cout<<"[pumi error] "<<__func__<<" not supported\n"; } int pumi_ment_getGlobalID(pMeshEnt e) diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 735e48ff2..81124ac04 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -231,41 +231,6 @@ pMesh pumi_mesh_load(pMesh m) return pumi::instance()->mesh; } - - - - - -/* -pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* mesh_type) -{ - if (strcmp(mesh_type,"mds")) - { - if (!PCU_Comm_Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<1) // do static partitioning - { - MPI_Comm prevComm = PCU_Get_Comm(); - int num_target_part = PCU_Comm_Peers()/num_in_part; - bool isMaster = ((PCU_Comm_Self() % num_target_part) == 0); - pMesh m = 0; - apf::Migration* plan = 0; - split_comm(num_target_part); - if (isMaster) { - m = apf::loadMdsMesh(g->getGmi(), filename); - plan = getPlan(m, num_target_part); - } - merge_comm(prevComm); - pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part); - } - else - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename); - pumi_mesh_print(pumi::instance()->mesh); - return pumi::instance()->mesh; -} -*/ - pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, pcu::PCU *PCUObj, const char* mesh_type) { if (strcmp(mesh_type,"mds")) @@ -273,7 +238,7 @@ pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, pcu::PCU *P if (!PCUObj->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<1) // do static partitioning + if (num_in_part==1 && PCUObj->Peers()>1) // do static partitioning { MPI_Comm prevComm = PCUObj->GetMPIComm(); int num_target_part = PCUObj->Peers()/num_in_part; @@ -317,37 +282,10 @@ void send_entities(pMesh mesh, int dim) #include "apfMDS.h" #include "apfPM.h" -/* -pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) -{ - if (pumi_size()==1) - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename); - else - { - double t0 = pcu::Time(); - MPI_Comm prevComm = PCU_Get_Comm(); - int num_target_part = PCU_Comm_Peers(); - split_comm(num_target_part); - // no pmodel & remote links setup - pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename); - merge_comm(prevComm); - if (!PCU_Comm_Self()) - lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); - } - - if (pumi_size()>1 && stitch_link) - { - stitchMesh(pumi::instance()->mesh); - pumi::instance()->mesh->acceptChanges(); - } - - return pumi::instance()->mesh; -} -*/ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool stitch_link) { - if (pumi_size(PCUObj)==1) + if (pumi_size()==1) pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); else { @@ -362,7 +300,7 @@ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool st lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); } - if (pumi_size(PCUObj)>1 && stitch_link) + if (pumi_size()>1 && stitch_link) { stitchMesh(pumi::instance()->mesh); pumi::instance()->mesh->acceptChanges(); @@ -413,7 +351,7 @@ void pumi_mesh_setCount(pMesh m, pOwnership o) } MPI_Allreduce(pumi::instance()->num_own_ent, pumi::instance()->num_global_ent, 4, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); #ifdef DEBUG - if (!pumi_rank(m->getPCU())) std::cout<<"[PUMI INFO] "<<__func__<<" end\n"; + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" end\n"; #endif } @@ -482,7 +420,7 @@ void pumi_mesh_print (pMesh m, bool print_ent) local_entity_count[i]=own_entity_count[i]=0; pMeshEnt e; - int self = pumi_rank(m->getPCU()); + int self = pumi_rank(); for (int d=0; d<4;++d) { @@ -491,7 +429,7 @@ void pumi_mesh_print (pMesh m, bool print_ent) while ((e = m->iterate(it))) { if (m->getOwner(e)==self) - ++own_entity_count[4*pumi_rank(m->getPCU())+d]; + ++own_entity_count[4*pumi_rank()+d]; } m->end(it); } @@ -716,7 +654,7 @@ void pumi_ownership_verify(pMesh m, pOwnership o) own_copy = pumi_ment_getOwnEnt(e,o); if (!own_copy) { - std::cout<<"[ERROR] ("<getPCU())<<") "<<__func__<<": pumi_ment_getOwnEnt(dim "<getPCU())<<") "<<__func__<<": pumi_ment_getOwnEnt(dim "<getPCU())<<") "<<__func__<<": pumi_ment_getRmt(dim "<getPCU())<<") "<<__func__<<": pumi_ment_getRmt and pumi_ment_getOwnEnt mismatch for e(dim "<getPCU())) std::cout<<__func__<<": ownership is valid\n"; + if (!pumi_rank()) std::cout<<__func__<<": ownership is valid\n"; } Distribution::Distribution(pMesh mesh) diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index 6ed34588f..870e2f88e 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -26,7 +26,7 @@ pNumbering pumi_numbering_create pNumbering n = m->findNumbering(name); if (n) { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<findNumbering(name); if (n) { - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" failed: numbering \""<Peers(); + return pumi::instance()->getPCU()->Peers(); } -int pumi_rank(pcu::PCU *PCUObj) +int pumi_rank() { - return PCUObj->Self(); + return pumi::instance()->getPCU()->Self(); } -void pumi_sync(pcu::PCU *PCUObj) +void pumi_sync() { - MPI_Barrier(PCUObj->GetMPIComm()); + MPI_Barrier(pumi::instance()->getPCU()->GetMPIComm()); } #include #include -void pumi_printSys(pcu::PCU *PCUObj) +void pumi_printSys() { - if (PCUObj->Self()) return; + if (pumi::instance()->getPCU()->Self()) return; struct utsname u; if (uname(&u) == 0) lion_oprint(1,"[%s] %s %s %s %s %s\n\n", @@ -61,9 +57,9 @@ double pumi_getMem() return pcu::GetMem(); } -void pumi_printTimeMem(const char* msg, double time, double memory, pcu::PCU *PCUObj) +void pumi_printTimeMem(const char* msg, double time, double memory) { - if (!PCUObj->Self()) + if (!pumi::instance()->getPCU()->Self()) { lion_oprint(1,"%-20s %6.3f sec %7.3f MB \n", msg, time, memory); fflush(stdout); diff --git a/test/cgns.cc b/test/cgns.cc index d92c69a9e..78098b519 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -482,6 +482,7 @@ int main(int argc, char **argv) MPI_Init(&argc, &argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pumi::instance()->setPCU(PCUObj.get()); lion_set_verbosity(1); bool additionalTests = false; if (argc < 3) diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index fcf8b908b..e4e4d16e0 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -16,6 +16,7 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pumi::instance()->setPCU(pcu_obj.get()); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -44,7 +45,7 @@ int main(int argc, char** argv) outMap.clear(); m->verify(); - if (!pumi_rank(pcu_obj.get())) printf("model/mesh converted to pumi instance\n"); + if (!pcu_obj.get()->Self()) printf("model/mesh converted to pumi instance\n"); //create the pumi instance to use pumi api's pGeom g = pumi_geom_load(model, pcu_obj.get()); diff --git a/test/pumi.cc b/test/pumi.cc index 9e13306df..9bfa82a36 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -90,8 +90,9 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pumi::instance()->setPCU(PCUObj.get()); lion_set_verbosity(1); - pumi_printSys(PCUObj.get()); + pumi_printSys(); #if 0 int i, processid = getpid(); @@ -111,7 +112,7 @@ int main(int argc, char** argv) // load model pGeom g = pumi_geom_load(modelFile, PCUObj.get()); - if (!pumi_rank(PCUObj.get())) std::cout<<"[test_pumi] testing geometric model/entity api's\n\n"; + if (!pumi_rank()) std::cout<<"[test_pumi] testing geometric model/entity api's\n\n"; { // test geom_find and gent_adj for (pGeomIter gent_it = g->begin(2); gent_it!=g->end(2);++gent_it) @@ -138,7 +139,7 @@ int main(int argc, char** argv) } // load mesh per process group - PCU_ALWAYS_ASSERT(pumi_size(PCUObj.get())%num_in_part==0); + PCU_ALWAYS_ASSERT(pumi_size()%num_in_part==0); double begin_mem = pumi_getMem(), begin_time=pumi_getTime(); @@ -182,7 +183,7 @@ int main(int argc, char** argv) m->end(it); pumi_mesh_distribute(m, plan); - if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] writing mesh in vtk\n"; + if (!pumi_rank()) std::cout<<"\n[test_pumi] writing mesh in vtk\n"; // write mesh in .smb pumi_mesh_write(m,outFile); @@ -197,17 +198,17 @@ int main(int argc, char** argv) pumi_mesh_delete(m); g = pumi_geom_load(modelFile, PCUObj.get()); - if (num_in_part==1 && pumi_size(PCUObj.get())>1) - m = pumi_mesh_load(g, "mesh.smb", pumi_size(PCUObj.get()), PCUObj.get()); + if (num_in_part==1 && pumi_size()>1) + m = pumi_mesh_load(g, "mesh.smb", pumi_size(), PCUObj.get()); else m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj.get()); - if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] delete and reload mesh\n"; + if (!pumi_rank()) std::cout<<"\n[test_pumi] delete and reload mesh\n"; pOwnership o=new testOwnership(m); pumi_ownership_verify(m, o); delete o; - if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] clean loaded tags from the mesh file\n"; + if (!pumi_rank()) std::cout<<"\n[test_pumi] clean loaded tags from the mesh file\n"; std::vector tag_vec; for (size_t n = 0; n::iterator fit=fields.begin(); fit!=fields.end(); ++fit) pumi_field_freeze(*fit); - if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] "<::iterator fit=fields.begin(); fit!=fields.end(); ++fit) pumi_field_delete(*fit); - if (!pumi_rank(PCUObj.get())) std::cout<<"\n[test_pumi] field and numbering deleted\n"; + if (!pumi_rank()) std::cout<<"\n[test_pumi] field and numbering deleted\n"; pumi_mesh_verify(m); TEST_MESH_TAG(m); @@ -249,7 +250,7 @@ int main(int argc, char** argv) pumi_mesh_delete(m); // print elapsed time and increased heap memory - pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem, PCUObj.get()); + pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem); } MPI_Finalize(); @@ -293,13 +294,13 @@ void TEST_MESH(pMesh m) for (pCopyIter it = copies.begin(); it != copies.end(); ++it) - PCU_ALWAYS_ASSERT(it->first!=pumi_rank(m->getPCU())); + PCU_ALWAYS_ASSERT(it->first!=pumi_rank()); // check the entity is not ghost or ghosted PCU_ALWAYS_ASSERT(!pumi_ment_isGhost(e) && !pumi_ment_isGhosted(e)); } m->end(mit); - if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] testing mesh/entity apis\n"; + if (!pumi_rank()) std::cout<<"\n[test_pumi] testing mesh/entity apis\n"; } #include @@ -676,7 +677,7 @@ void TEST_NEW_MESH(pMesh m) // pumi_mesh_freeze(new_m); - if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] new mesh constructed (#v " + if (!pumi_rank()) std::cout<<"\n[test_pumi] new mesh constructed (#v " <iterate(it))) { - for (int i=0; igetPCU())/2; ++i) + for (int i=0; igetPCU()); + int pid = rand()%pumi_size(); plan->send(e, pid); } ++count; @@ -792,7 +793,7 @@ void TEST_GHOSTING(pMesh m) int total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] element-wise pumi_ghost_create: #ghost increase="<begin(0); @@ -802,7 +803,7 @@ void TEST_GHOSTING(pMesh m) if (pumi_ment_isGhost(e)) { ++num_ghost_vtx; - PCU_ALWAYS_ASSERT(pumi_ment_getOwnPID(e)!=pumi_rank(m->getPCU())); + PCU_ALWAYS_ASSERT(pumi_ment_getOwnPID(e)!=pumi_rank()); } } m->end(mit); @@ -823,7 +824,7 @@ void TEST_GHOSTING(pMesh m) pumi_ghost_createLayer (m, brg_dim, mesh_dim, num_layer, include_copy); total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - if (!pumi_rank(m->getPCU())) std::cout<<"\n[test_pumi] layer-wise pumi_ghost_createLayer (bd "<getPCU()->GetMPIComm()); - if (!pumi_rank(m->getPCU())) + if (!pumi_rank()) std::cout<<"\n[test_pumi] accumulative pumi_ghost_createLayer (bd "<getPCU())<<") ERROR dim "<(new pcu::PCU(MPI_COMM_WORLD)); + pumi::instance()->setPCU(PCUObj.get()); pGeom g = pumi_geom_load(argv[1], PCUObj.get(), "mesh"); pMesh m = pumi_mesh_load(g, argv[2], 1, PCUObj.get()); pumi_mesh_delete(m); diff --git a/test/xgc_split.cc b/test/xgc_split.cc index e69cb2e44..3cf0898c3 100644 --- a/test/xgc_split.cc +++ b/test/xgc_split.cc @@ -18,7 +18,7 @@ int serial=0; void getConfig(int argc, char** argv, pcu::PCU* PCUObj) { if (argc < 4) { - if (!pumi_rank(PCUObj) ) + if (!PCUObj->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -34,11 +34,11 @@ Migration* get_xgc_plan(pGeom g, pMesh m) { int dim = pumi_mesh_getDim(m); Migration* plan = new Migration(m); - if (!pumi_rank(m->getPCU())) return plan; + if (!pumi_rank()) return plan; pMeshEnt e; int num_gface = pumi_geom_getNumEnt(g, dim); - PCU_ALWAYS_ASSERT(num_gface==pumi_size(m->getPCU())); + PCU_ALWAYS_ASSERT(num_gface==pumi_size()); int gface_id; int dest_pid; pMeshIter it = m->begin(2); // face @@ -72,7 +72,7 @@ int main(int argc, char** argv) pumi_mesh_write(m, outFile); } else - m = pumi_mesh_load(g, meshFile, pumi_size(PCUObj.get()), PCUObj.get()); + m = pumi_mesh_load(g, meshFile, pumi_size(), PCUObj.get()); // write to vtk char without_extension[256]; From 687e160dd60ca2e7bd133a8a7f986965b802c28a Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 9 Jul 2024 13:12:08 -0400 Subject: [PATCH 120/141] added pumi_load_pcu to replace the use of setPCU, removed PCUObj as an argument for pumi mesh creators --- pumi/pumi.h | 20 ++++++++----- pumi/pumi_geom.cc | 12 ++++---- pumi/pumi_mesh.cc | 58 +++++++++++++++++++------------------- pumi/pumi_sys.cc | 4 ++- test/cgns.cc | 2 +- test/constructThenGhost.cc | 4 +-- test/pumi.cc | 18 ++++++------ test/pumiLoadMesh.cc | 6 ++-- test/xgc_split.cc | 8 +++--- 9 files changed, 70 insertions(+), 62 deletions(-) diff --git a/pumi/pumi.h b/pumi/pumi.h index b4e58494d..3a3cd91fc 100644 --- a/pumi/pumi.h +++ b/pumi/pumi.h @@ -15,6 +15,7 @@ #include "GenIterator.h" #include "mPartEntityContainer.h" #include "apf.h" +#include "pcu_util.h" enum PUMI_EntTopology { PUMI_VERTEX, // 0 @@ -90,7 +91,10 @@ class pumi static pumi _instance; return &_instance; }; - void setPCU(pcu::PCU *newPCU) {PCUObj = newPCU;} + void initializePCU(pcu::PCU *newPCU) { + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj==nullptr, "pumi::instance() PCUObj already initialized\n"); + PCUObj = newPCU; + } [[nodiscard]] pcu::PCU* getPCU() const noexcept {return PCUObj;} pMesh mesh; @@ -102,6 +106,7 @@ class pumi pMeshTag ghost_tag; std::vector ghost_vec[4]; std::vector ghosted_vec[4]; + private: pcu::PCU *PCUObj; }; @@ -112,6 +117,7 @@ class pumi //************************************ //************************************ +void pumi_load_pcu(pcu::PCU *PCUObj); int pumi_size(); int pumi_rank(); @@ -129,12 +135,12 @@ void pumi_printTimeMem(const char* msg, double time, double memory); // Geometric Model // create a model from gmi_model object -pGeom pumi_geom_load(gmi_model* gm, pcu::PCU *PCUObj, +pGeom pumi_geom_load(gmi_model* gm, const char* model_type="mesh", const char* fileName=NULL, void (*fp)(const char*)=NULL); // create a model from a file -pGeom pumi_geom_load (const char* fileName, pcu::PCU *PCUObj, const char* model_type="mesh", +pGeom pumi_geom_load (const char* fileName, const char* model_type="mesh", void (*fp)(const char*)=NULL); void pumi_geom_delete(pGeom g); void pumi_geom_freeze(pGeom g); // shall be called after modifying model entities @@ -200,7 +206,7 @@ void pumi_gent_getEntArrTag (pGeomEnt ent, pTag tag, pGeomEnt** data, int* data_ //************************************ // create an empty mesh -pMesh pumi_mesh_create(pGeom g, int mesh_dim, pcu::PCU *PCUObj, bool periodic=false); +pMesh pumi_mesh_create(pGeom g, int mesh_dim, bool periodic=false); void pumi_mesh_freeze(pMesh m); pMeshEnt pumi_mesh_createVtx(pMesh m, pGeomEnt ge, double* xyz); //ent_topology: VERTEX (0), EDGE (1), TRIANGLE (2), QUAD (3), TET (4), HEX (5), PRISM (6), PYRAMID (7) @@ -208,17 +214,17 @@ pMeshEnt pumi_mesh_createEnt(pMesh m, pGeomEnt ge, int target_topology, pMeshEnt pMeshEnt pumi_mesh_createElem(pMesh m, pGeomEnt ge, int target_topology, pMeshEnt* vertices); // load a serial mesh and no partitioning -pMesh pumi_mesh_loadSerial(pGeom g, const char* file_name, pcu::PCU* PCUObj, const char* mesh_type="mds"); +pMesh pumi_mesh_loadSerial(pGeom g, const char* file_name, const char* mesh_type="mds"); // load a mesh from a file. Do static partitioning if num_in_part==1 -pMesh pumi_mesh_load(pGeom geom, const char* fileName, int num_in_part, pcu::PCU* PCUObj, const char* mesh_type="mds"); +pMesh pumi_mesh_load(pGeom geom, const char* fileName, int num_in_part, const char* mesh_type="mds"); // load a mesh from a an existing partitioned apf mesh pMesh pumi_mesh_load(pMesh mesh); // load a serial mesh on all processes and set up comm links and ptn classification // note that the default owning PID is 0 -pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool stich_link=true); +pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stich_link=true); // delete mesh void pumi_mesh_delete(pMesh m); diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index ef2f1239f..331146222 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -31,27 +31,27 @@ gEntity* gModel::getGeomEnt(int d, gmi_ent* ge) return allEntities.getGeomEnt(d, ge); } -pGeom pumi_geom_load(const char* filename, pcu::PCU *PCUObj, const char* model_type, void (*geom_load_fp)(const char*)) +pGeom pumi_geom_load(const char* filename, const char* model_type, void (*geom_load_fp)(const char*)) { if (!strcmp(model_type,"null")) { gmi_register_null(); - return pumi_geom_load(gmi_load(".null"), PCUObj, model_type); + return pumi_geom_load(gmi_load(".null"), model_type); } else if (!strcmp(model_type,"mesh")) { gmi_register_mesh(); - return pumi_geom_load(gmi_load(filename), PCUObj); + return pumi_geom_load(gmi_load(filename)); } else if (!strcmp(model_type,"analytic")) - return pumi_geom_load(gmi_make_analytic(), PCUObj, model_type, filename, geom_load_fp); + return pumi_geom_load(gmi_make_analytic(), model_type, filename, geom_load_fp); else if (!pumi_rank()) lion_eprint(1,"[PUMI ERROR] unsupported model type %s\n",model_type); return NULL; } -pGeom pumi_geom_load(gmi_model* gm, pcu::PCU *PCUObj, const char* model_type, +pGeom pumi_geom_load(gmi_model* gm, const char* model_type, const char* filename, void (*geom_load_fp)(const char*)) { double t0 = pcu::Time(); @@ -77,7 +77,7 @@ pGeom pumi_geom_load(gmi_model* gm, pcu::PCU *PCUObj, const char* model_type, return NULL; } - if (!PCUObj->Self() && filename) + if (!pumi::instance()->getPCU()->Self() && filename) lion_oprint(1,"model %s loaded in %f seconds\n", filename, pcu::Time() - t0); return pumi::instance()->model; diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 81124ac04..b20051b40 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -28,9 +28,9 @@ using std::map; // mesh creation -pMesh pumi_mesh_create(pGeom g, int mesh_dim, pcu::PCU *PCUObj, bool periodic) +pMesh pumi_mesh_create(pGeom g, int mesh_dim, bool periodic) { - pumi::instance()->mesh = apf::makeEmptyMdsMesh(g->getGmi(), mesh_dim, periodic, PCUObj); + pumi::instance()->mesh = apf::makeEmptyMdsMesh(g->getGmi(), mesh_dim, periodic, pumi::instance()->getPCU()); return pumi::instance()->mesh; } @@ -205,21 +205,21 @@ pGeom pumi_mesh_getGeom(pMesh) -pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, pcu::PCU *PCUObj, const char* mesh_type) +pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, const char* mesh_type) { if (strcmp(mesh_type,"mds")) { - if (!PCUObj->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<GetMPIComm(); - int num_target_part = PCUObj->Peers(); - bool isMaster = ((PCUObj->Self() % num_target_part) == 0); + MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); + int num_target_part = pumi::instance()->getPCU()->Peers(); + bool isMaster = ((pumi::instance()->getPCU()->Self() % num_target_part) == 0); pMesh m = 0; - split_comm(num_target_part, *PCUObj); + split_comm(num_target_part, *pumi::instance()->getPCU()); if (isMaster) - m = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); - merge_comm(prevComm, *PCUObj); + m = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); + merge_comm(prevComm, *pumi::instance()->getPCU()); pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1, m->getPCU()); return pumi::instance()->mesh; } @@ -231,30 +231,30 @@ pMesh pumi_mesh_load(pMesh m) return pumi::instance()->mesh; } -pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, pcu::PCU *PCUObj, const char* mesh_type) +pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* mesh_type) { if (strcmp(mesh_type,"mds")) { - if (!PCUObj->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<Peers()>1) // do static partitioning + if (num_in_part==1 && pumi::instance()->getPCU()->Peers()>1) // do static partitioning { - MPI_Comm prevComm = PCUObj->GetMPIComm(); - int num_target_part = PCUObj->Peers()/num_in_part; - bool isMaster = ((PCUObj->Self() % num_target_part) == 0); + MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); + int num_target_part = pumi::instance()->getPCU()->Peers()/num_in_part; + bool isMaster = ((pumi::instance()->getPCU()->Self() % num_target_part) == 0); pMesh m = 0; apf::Migration* plan = 0; - split_comm(num_target_part, *PCUObj); + split_comm(num_target_part, *pumi::instance()->getPCU()); if (isMaster) { - m = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + m = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); plan = getPlan(m, num_target_part); } - merge_comm(prevComm, *PCUObj); - pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part, PCUObj); + merge_comm(prevComm, *pumi::instance()->getPCU()); + pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part, pumi::instance()->getPCU()); } else - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); pumi_mesh_print(pumi::instance()->mesh); return pumi::instance()->mesh; } @@ -283,20 +283,20 @@ void send_entities(pMesh mesh, int dim) #include "apfMDS.h" #include "apfPM.h" -pMesh pumi_mesh_loadAll(pGeom g, const char* filename, pcu::PCU *PCUObj, bool stitch_link) +pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) { if (pumi_size()==1) - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, PCUObj); + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); else { double t0 = pcu::Time(); - MPI_Comm prevComm = PCUObj->GetMPIComm(); - int num_target_part = PCUObj->Peers(); - split_comm(num_target_part, *PCUObj); + MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); + int num_target_part = pumi::instance()->getPCU()->Peers(); + split_comm(num_target_part, *pumi::instance()->getPCU()); // no pmodel & remote links setup - pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename, PCUObj); - merge_comm(prevComm, *PCUObj); - if (!PCUObj->Self()) + pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); + merge_comm(prevComm, *pumi::instance()->getPCU()); + if (!pumi::instance()->getPCU()->Self()) lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); } diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc index bbdde190b..04f128bc9 100644 --- a/pumi/pumi_sys.cc +++ b/pumi/pumi_sys.cc @@ -16,7 +16,9 @@ // 0- SYSTEM-LEVEL FUNCTIONS //************************************ //************************************ - +void pumi_load_pcu(pcu::PCU *PCUObj){ + pumi::instance()->initializePCU(PCUObj); +} int pumi_size() { diff --git a/test/cgns.cc b/test/cgns.cc index 78098b519..316beb927 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -482,7 +482,7 @@ int main(int argc, char **argv) MPI_Init(&argc, &argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi::instance()->setPCU(PCUObj.get()); + pumi_load_pcu(PCUObj.get()); lion_set_verbosity(1); bool additionalTests = false; if (argc < 3) diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index e4e4d16e0..95e5e38f0 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -16,7 +16,7 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi::instance()->setPCU(pcu_obj.get()); + pumi_load_pcu(pcu_obj.get()); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -48,7 +48,7 @@ int main(int argc, char** argv) if (!pcu_obj.get()->Self()) printf("model/mesh converted to pumi instance\n"); //create the pumi instance to use pumi api's - pGeom g = pumi_geom_load(model, pcu_obj.get()); + pGeom g = pumi_geom_load(model); pMesh pm = pumi_mesh_load(m); pumi_mesh_verify(pm); diff --git a/test/pumi.cc b/test/pumi.cc index 9bfa82a36..320254d99 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -90,7 +90,7 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi::instance()->setPCU(PCUObj.get()); + pumi_load_pcu(PCUObj.get()); lion_set_verbosity(1); pumi_printSys(); @@ -110,7 +110,7 @@ int main(int argc, char** argv) getConfig(argc,argv,PCUObj.get()); // load model - pGeom g = pumi_geom_load(modelFile, PCUObj.get()); + pGeom g = pumi_geom_load(modelFile); if (!pumi_rank()) std::cout<<"[test_pumi] testing geometric model/entity api's\n\n"; { @@ -145,10 +145,10 @@ int main(int argc, char** argv) pMesh m=NULL; if (do_distr) - m = pumi_mesh_loadSerial(g, meshFile, PCUObj.get()); + m = pumi_mesh_loadSerial(g, meshFile); else { - m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj.get()); // static partitioning if num_in_part=1 + m = pumi_mesh_load(g, meshFile, num_in_part); // static partitioning if num_in_part=1 if (num_in_part==1) pumi_mesh_write(m,"mesh.smb"); } @@ -197,11 +197,11 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(m); - g = pumi_geom_load(modelFile, PCUObj.get()); + g = pumi_geom_load(modelFile); if (num_in_part==1 && pumi_size()>1) - m = pumi_mesh_load(g, "mesh.smb", pumi_size(), PCUObj.get()); + m = pumi_mesh_load(g, "mesh.smb", pumi_size()); else - m = pumi_mesh_load(g, meshFile, num_in_part, PCUObj.get()); + m = pumi_mesh_load(g, meshFile, num_in_part); if (!pumi_rank()) std::cout<<"\n[test_pumi] delete and reload mesh\n"; pOwnership o=new testOwnership(m); @@ -638,8 +638,8 @@ void TEST_NEW_MESH(pMesh m) PCU_ALWAYS_ASSERT(pumi_shape_getNumNode(pumi_mesh_getShape(m), 1)==1); // create an empty mesh - pGeom new_g = pumi_geom_load("", m->getPCU(), "null"); - pMesh new_m = pumi_mesh_create(new_g, 2, m->getPCU()); + pGeom new_g = pumi_geom_load("", "null"); + pMesh new_m = pumi_mesh_create(new_g, 2); double xyz[3]; pMeshIter it = m->begin(1); diff --git a/test/pumiLoadMesh.cc b/test/pumiLoadMesh.cc index f36525a13..5893f95f0 100644 --- a/test/pumiLoadMesh.cc +++ b/test/pumiLoadMesh.cc @@ -7,9 +7,9 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi::instance()->setPCU(PCUObj.get()); - pGeom g = pumi_geom_load(argv[1], PCUObj.get(), "mesh"); - pMesh m = pumi_mesh_load(g, argv[2], 1, PCUObj.get()); + pumi_load_pcu(PCUObj.get()); + pGeom g = pumi_geom_load(argv[1], "mesh"); + pMesh m = pumi_mesh_load(g, argv[2], 1); pumi_mesh_delete(m); pumi_geom_delete(g); } diff --git a/test/xgc_split.cc b/test/xgc_split.cc index 3cf0898c3..67d4ef20c 100644 --- a/test/xgc_split.cc +++ b/test/xgc_split.cc @@ -58,21 +58,21 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - + pumi_load_pcu(PCUObj.get()); getConfig(argc,argv,PCUObj.get()); - pGeom g = pumi_geom_load(modelFile, PCUObj.get()); + pGeom g = pumi_geom_load(modelFile); pMesh m; if (serial) { - m = pumi_mesh_loadSerial(g, meshFile, PCUObj.get()); + m = pumi_mesh_loadSerial(g, meshFile); // split a serial mesh based on model ID Migration* plan = get_xgc_plan(g, m); pumi_mesh_migrate(m, plan); pumi_mesh_write(m, outFile); } else - m = pumi_mesh_load(g, meshFile, pumi_size(), PCUObj.get()); + m = pumi_mesh_load(g, meshFile, pumi_size()); // write to vtk char without_extension[256]; From 7d57abceaf7933657410f3d370c79ac28e4b85ce Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 31 Jul 2024 15:50:00 -0400 Subject: [PATCH 121/141] apfSim typo fix --- apf_sim/apfSIM.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apf_sim/apfSIM.cc b/apf_sim/apfSIM.cc index 2d2ead446..09a447a19 100644 --- a/apf_sim/apfSIM.cc +++ b/apf_sim/apfSIM.cc @@ -1092,7 +1092,7 @@ static bool findMatches(Mesh* m) return found; } -Mesh2* createMesh(pParMesh mesh, pcu::PCU PCUObj) +Mesh2* createMesh(pParMesh mesh, pcu::PCU *PCUObj) { /* require one part per process currently for SIM */ PCU_ALWAYS_ASSERT(PM_numParts(mesh)==1); From e90602f54454527c349fdc7a96b70d5cf0e09101 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 31 Jul 2024 16:08:26 -0400 Subject: [PATCH 122/141] pumi_rank() replacements --- pumi/pumi_geom.cc | 4 ++-- test/constructThenGhost.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index 331146222..47b407bf0 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -77,7 +77,7 @@ pGeom pumi_geom_load(gmi_model* gm, const char* model_type, return NULL; } - if (!pumi::instance()->getPCU()->Self() && filename) + if (!pumi_rank() && filename) lion_oprint(1,"model %s loaded in %f seconds\n", filename, pcu::Time() - t0); return pumi::instance()->model; @@ -193,7 +193,7 @@ void pumi_giter_reset(gIter iter) void pumi_geom_print (pGeom g, bool print_ent) { - if (pumi::instance()->getPCU()->Self()) return; + if (pumi_rank()) return; std::cout<<"\n=== model entity and tag info === \n"; std::cout<<"# global geom ent: v "<size(0)<<", e " <size(1)<<", f "<size(2)<<", r "<size(3)<<"\n"; diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index 95e5e38f0..66b8e1ac4 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -45,7 +45,7 @@ int main(int argc, char** argv) outMap.clear(); m->verify(); - if (!pcu_obj.get()->Self()) printf("model/mesh converted to pumi instance\n"); + if (!pumi_rank()) printf("model/mesh converted to pumi instance\n"); //create the pumi instance to use pumi api's pGeom g = pumi_geom_load(model); From 55f887c33bcc2a13ce68ee3cb3fbac978eab74cd Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 6 Aug 2024 11:54:40 -0400 Subject: [PATCH 123/141] simmetrix create mesh typo fixed --- phasta/ph.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phasta/ph.cc b/phasta/ph.cc index e1fcccf21..417122b61 100644 --- a/phasta/ph.cc +++ b/phasta/ph.cc @@ -166,7 +166,7 @@ apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj) { pGModel simModel = gmi_export_sim(g); pParMesh sim_mesh = PM_load(meshfile, simModel, progress); - mesh = apf::createMesh(sim_mesh); + mesh = apf::createMesh(sim_mesh, PCUObj); Progress_delete(progress); } else From a904f0270d54d14aa329c821e910c8c8ab8b3e16 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 7 Aug 2024 12:25:03 -0400 Subject: [PATCH 124/141] phLinks ptr issue fix --- phasta/phLinks.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/phasta/phLinks.cc b/phasta/phLinks.cc index 73a2a0ff3..62b72f7d1 100644 --- a/phasta/phLinks.cc +++ b/phasta/phLinks.cc @@ -3,6 +3,7 @@ #include #include #include +#include namespace ph { @@ -23,7 +24,7 @@ struct PhastaSharing : public apf::Sharing { helperN = new apf::NormalSharing(m); helperM = new apf::MatchedSharing(m); } - ~PhastaSharing() + virtual ~PhastaSharing() { delete helperN; delete helperM; @@ -97,21 +98,22 @@ struct PhastaSharing : public apf::Sharing { void getLinks(apf::Mesh* m, int dim, Links& links, BCs& bcs) { - PhastaSharing shr(m); + //PhastaSharing* shr = new PhastaSharing(m); + auto shr = std::unique_ptr(new PhastaSharing(m)); m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(dim); apf::MeshEntity* v; while ((v = m->iterate(it))) { apf::ModelEntity* me = m->toModel(v); - shr.isDG = ph::isInterface(m->getModel(),(gmi_ent*) me,bcs.fields["DG interface"]); + shr->isDG = ph::isInterface(m->getModel(),(gmi_ent*) me,bcs.fields["DG interface"]); /* the alignment is such that the owner part's array follows the order of its vertex iterator traversal. The owner dictates the order to the other part by sending remote copies */ - if ( ! shr.isOwned(v)) + if ( ! shr->isOwned(v)) continue; apf::CopyArray remotes; - shr.getCopies(v, remotes); + shr->getCopies(v, remotes); for (size_t i = 0; i < remotes.getSize(); ++i) { /* in matching we may accumulate multiple occurrences of the same master in the outgoing links array From 77812e51cab092651f86c9ca7f842292ecda6889 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 7 Aug 2024 15:09:40 -0400 Subject: [PATCH 125/141] phasta convert fixes --- phasta/ph_convert.cc | 2 +- test/convert.cc | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 16b9bc3f2..e966fc8c8 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -265,7 +265,7 @@ int main(int argc, char** argv) M_release(sim_mesh); postConvert(mesh); mesh->writeNative(smb_path); - std::string restartPath = ph::setupOutputDir(); + std::string restartPath = ph::setupOutputDir(mesh->getPCU()); ph::Input phIn; phIn.openfile_read = openFileRead; ph::Output phOut; diff --git a/test/convert.cc b/test/convert.cc index 0859ef2f5..50a1c1ef7 100644 --- a/test/convert.cc +++ b/test/convert.cc @@ -120,18 +120,18 @@ void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { gmi_native_path = optarg; break; case '?': - if (!pcu_obj.get()->Self()) + if (!pcu_obj->Self()) printf ("warning: skipping unrecognized option \'%s\'\n", argv[optind-1]); break; default: - if (!pcu_obj.get()->Self()) + if (!pcu_obj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!pcu_obj.get()->Self()) + if (!pcu_obj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -139,7 +139,7 @@ void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { gmi_path = argv[i++]; sms_path = argv[i++]; smb_path = argv[i++]; - if (!pcu_obj.get()->Self()) { + if (!pcu_obj->Self()) { printf ("fix_pyramids %d attach_order %d enable_log %d extruRootPath %s\n", should_fix_pyramids, should_attach_order, should_log, extruRootPath); printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", @@ -359,7 +359,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int* fatherIdPtr; const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr); if(!exists) { - if(!PCU_Comm_Self()) + if(!simApfMesh->getPCU()->Self()) fprintf(stderr, "Error: father id data pointer does not exist... exiting\n"); exit(EXIT_FAILURE); } From a907b7e0a21c5acfafce18b8a5b2e3be90bfe40f Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 8 Aug 2024 17:05:42 -0400 Subject: [PATCH 126/141] test files small bugs --- config.sh | 15 +++++++++++++++ test/generate.cc | 8 +++++--- test/highOrderSolutionTransfer.cc | 2 +- test/sim_part.cc | 2 ++ 4 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 config.sh diff --git a/config.sh b/config.sh new file mode 100644 index 000000000..5f73e72aa --- /dev/null +++ b/config.sh @@ -0,0 +1,15 @@ +cmake -S . -B /lore/mccalf/build \ + -DCMAKE_C_COMPILER=mpicc \ + -DCMAKE_CXX_COMPILER=mpicxx \ + -DSCOREC_CXX_WARNINGS=on \ + -DIS_TESTING=on \ + -DBUILD_EXES=on \ + -DMESHES=/users/mccalf/pcu-update/pumi-meshes \ + -DENABLE_ZOLTAN=on \ + -DSIM_MPI=mpich4.1.1 \ + -DSIM_PARASOLID=on \ + -DSIM_ACIS=on \ + -DENABLE_SIMMETRIX=on \ + -DSKIP_SIMMETRIX_VERSION_CHECK=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_INSTALL_PREFIX=/lore/mccalf/coreInstall/ diff --git a/test/generate.cc b/test/generate.cc index 9d0ea10ce..1a8b31ffb 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -42,6 +42,7 @@ #include "SimAttribute.h" #include "ModelTypes.h" +static const pcu::PCU* globalPCU = nullptr; namespace { @@ -60,11 +61,11 @@ void messageHandler(int type, const char* msg) { switch (type) { case Sim_WarningMsg: - if(!PCU_Comm_Self()) + if(!globalPCU->Self()) fprintf(stdout, "Warning SimModeler %s\n", msg); break; case Sim_ErrorMsg: - if(!PCU_Comm_Self()) + if(!globalPCU->Self()) fprintf(stdout, "Error SimModeler %s ... exiting\n", msg); MPI_Finalize(); exit(EXIT_SUCCESS); @@ -76,11 +77,11 @@ void messageHandler(int type, const char* msg) } pParMesh generate(pGModel mdl, std::string meshCaseName, pcu::PCU *PCUObj) { - pAManager attmngr = SModel_attManager(mdl); #if SIMMODSUITE_MAJOR_VERSION <= 2024 && SIMMODSUITE_MINOR_VERSION < 240219 pAManager attmngr = GM_attManager(mdl); #else pAManager attmngr = GM_attManager(mdl,true); +#endif if(0==PCUObj->Self()) fprintf(stdout, "Loading mesh case %s...\n", meshCaseName.c_str()); pACase mcaseFile = AMAN_findCase(attmngr, meshCaseName.c_str()); @@ -323,6 +324,7 @@ int main(int argc, char** argv) MPI_Init(&argc, &argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + globalPCU = PCUObj.get(); lion_set_verbosity(1); pcu::Protect(); getConfig(argc,argv,PCUObj.get()); diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index f5ad96c41..430bae19a 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -80,7 +80,7 @@ int main(int argc, char** argv) 6 /*field_order*/); // cubic adapt - testCurveAdapt(modelFile, meshFile, PCUObj.get() + testCurveAdapt(modelFile, meshFile, PCUObj.get(), 3 /*mesh_order*/, 2 /*exact_order*/, 6 /*field_order*/); diff --git a/test/sim_part.cc b/test/sim_part.cc index 811f8abd4..16c96ef90 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -39,6 +39,8 @@ #endif #endif +#include +#include #include #include #include From 8128e2a64f03ad3bbd2332af8e9e688f4b463fa5 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 8 Aug 2024 17:21:02 -0400 Subject: [PATCH 127/141] Revert "test files small bugs" This reverts commit a907b7e0a21c5acfafce18b8a5b2e3be90bfe40f. --- test/generate.cc | 8 +++----- test/highOrderSolutionTransfer.cc | 2 +- test/sim_part.cc | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/generate.cc b/test/generate.cc index 1a8b31ffb..9d0ea10ce 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -42,7 +42,6 @@ #include "SimAttribute.h" #include "ModelTypes.h" -static const pcu::PCU* globalPCU = nullptr; namespace { @@ -61,11 +60,11 @@ void messageHandler(int type, const char* msg) { switch (type) { case Sim_WarningMsg: - if(!globalPCU->Self()) + if(!PCU_Comm_Self()) fprintf(stdout, "Warning SimModeler %s\n", msg); break; case Sim_ErrorMsg: - if(!globalPCU->Self()) + if(!PCU_Comm_Self()) fprintf(stdout, "Error SimModeler %s ... exiting\n", msg); MPI_Finalize(); exit(EXIT_SUCCESS); @@ -77,11 +76,11 @@ void messageHandler(int type, const char* msg) } pParMesh generate(pGModel mdl, std::string meshCaseName, pcu::PCU *PCUObj) { + pAManager attmngr = SModel_attManager(mdl); #if SIMMODSUITE_MAJOR_VERSION <= 2024 && SIMMODSUITE_MINOR_VERSION < 240219 pAManager attmngr = GM_attManager(mdl); #else pAManager attmngr = GM_attManager(mdl,true); -#endif if(0==PCUObj->Self()) fprintf(stdout, "Loading mesh case %s...\n", meshCaseName.c_str()); pACase mcaseFile = AMAN_findCase(attmngr, meshCaseName.c_str()); @@ -324,7 +323,6 @@ int main(int argc, char** argv) MPI_Init(&argc, &argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - globalPCU = PCUObj.get(); lion_set_verbosity(1); pcu::Protect(); getConfig(argc,argv,PCUObj.get()); diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index 430bae19a..f5ad96c41 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -80,7 +80,7 @@ int main(int argc, char** argv) 6 /*field_order*/); // cubic adapt - testCurveAdapt(modelFile, meshFile, PCUObj.get(), + testCurveAdapt(modelFile, meshFile, PCUObj.get() 3 /*mesh_order*/, 2 /*exact_order*/, 6 /*field_order*/); diff --git a/test/sim_part.cc b/test/sim_part.cc index 16c96ef90..811f8abd4 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -39,8 +39,6 @@ #endif #endif -#include -#include #include #include #include From 94549c7a0ba0a4afb35da44813a971798c351ad6 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 8 Aug 2024 17:44:09 -0400 Subject: [PATCH 128/141] removed config file --- test/generate.cc | 8 +++++--- test/highOrderSolutionTransfer.cc | 2 +- test/sim_part.cc | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/generate.cc b/test/generate.cc index 9d0ea10ce..1a8b31ffb 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -42,6 +42,7 @@ #include "SimAttribute.h" #include "ModelTypes.h" +static const pcu::PCU* globalPCU = nullptr; namespace { @@ -60,11 +61,11 @@ void messageHandler(int type, const char* msg) { switch (type) { case Sim_WarningMsg: - if(!PCU_Comm_Self()) + if(!globalPCU->Self()) fprintf(stdout, "Warning SimModeler %s\n", msg); break; case Sim_ErrorMsg: - if(!PCU_Comm_Self()) + if(!globalPCU->Self()) fprintf(stdout, "Error SimModeler %s ... exiting\n", msg); MPI_Finalize(); exit(EXIT_SUCCESS); @@ -76,11 +77,11 @@ void messageHandler(int type, const char* msg) } pParMesh generate(pGModel mdl, std::string meshCaseName, pcu::PCU *PCUObj) { - pAManager attmngr = SModel_attManager(mdl); #if SIMMODSUITE_MAJOR_VERSION <= 2024 && SIMMODSUITE_MINOR_VERSION < 240219 pAManager attmngr = GM_attManager(mdl); #else pAManager attmngr = GM_attManager(mdl,true); +#endif if(0==PCUObj->Self()) fprintf(stdout, "Loading mesh case %s...\n", meshCaseName.c_str()); pACase mcaseFile = AMAN_findCase(attmngr, meshCaseName.c_str()); @@ -323,6 +324,7 @@ int main(int argc, char** argv) MPI_Init(&argc, &argv); { auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + globalPCU = PCUObj.get(); lion_set_verbosity(1); pcu::Protect(); getConfig(argc,argv,PCUObj.get()); diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index f5ad96c41..430bae19a 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -80,7 +80,7 @@ int main(int argc, char** argv) 6 /*field_order*/); // cubic adapt - testCurveAdapt(modelFile, meshFile, PCUObj.get() + testCurveAdapt(modelFile, meshFile, PCUObj.get(), 3 /*mesh_order*/, 2 /*exact_order*/, 6 /*field_order*/); diff --git a/test/sim_part.cc b/test/sim_part.cc index 811f8abd4..16c96ef90 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -39,6 +39,8 @@ #endif #endif +#include +#include #include #include #include From 73c503341f26ceac10b004a0f1e22934779a4af5 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Fri, 16 Aug 2024 12:41:42 -0400 Subject: [PATCH 129/141] PCU.h small fix --- test/generate.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/generate.cc b/test/generate.cc index 1a8b31ffb..a34f0f6cc 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include From 90f9ff32e13a79d58c746fbcf8c103cde62d1ff0 Mon Sep 17 00:00:00 2001 From: Flynn McCallum <102677191+flagdanger@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:56:52 -0400 Subject: [PATCH 130/141] Delete config.sh --- config.sh | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 config.sh diff --git a/config.sh b/config.sh deleted file mode 100644 index 5f73e72aa..000000000 --- a/config.sh +++ /dev/null @@ -1,15 +0,0 @@ -cmake -S . -B /lore/mccalf/build \ - -DCMAKE_C_COMPILER=mpicc \ - -DCMAKE_CXX_COMPILER=mpicxx \ - -DSCOREC_CXX_WARNINGS=on \ - -DIS_TESTING=on \ - -DBUILD_EXES=on \ - -DMESHES=/users/mccalf/pcu-update/pumi-meshes \ - -DENABLE_ZOLTAN=on \ - -DSIM_MPI=mpich4.1.1 \ - -DSIM_PARASOLID=on \ - -DSIM_ACIS=on \ - -DENABLE_SIMMETRIX=on \ - -DSKIP_SIMMETRIX_VERSION_CHECK=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_INSTALL_PREFIX=/lore/mccalf/coreInstall/ From 236bffbf0af1597b4f766f102dfb42d56e43aae8 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 19 Aug 2024 10:50:49 -0400 Subject: [PATCH 131/141] long dereference chains fixed --- .github/workflows/cmake.yml | 2 +- ma/maRefine.cc | 9 +++++---- zoltan/apfZoltanCallbacks.cc | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 70c3f13b9..39ed4a701 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -36,7 +36,7 @@ jobs: env: MPICH_CXX: ${{matrix.compiler.CXX}} MPICH_CC: ${{matrix.compiler.CC}} - run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j 1 + run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} -j - name: Test env: diff --git a/ma/maRefine.cc b/ma/maRefine.cc index b437e45c5..915d8481a 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -245,11 +245,11 @@ void splitElement(Refine* r, Entity* e) static void linkNewVerts(Refine* r) { - if (r->adapt->mesh->getPCU()->Peers()==1) - return; - struct { Entity* parent; Entity* vert; } message; Adapt* a = r->adapt; Mesh* m = a->mesh; + if (m->getPCU()->Peers()==1) + return; + struct { Entity* parent; Entity* vert; } message; m->getPCU()->Begin(); for (int d=1; d < m->getDimension(); ++d) for (size_t i=0; i < r->newEntities[d].getSize(); ++i) @@ -402,7 +402,8 @@ long markEdgesToSplit(Adapt* a) void processNewElements(Refine* r) { linkNewVerts(r); - if (r->adapt->mesh->getPCU()->Peers()>1) { + Mesh* m = r->adapt->mesh; + if (m->getPCU()->Peers()>1) { apf::stitchMesh(r->adapt->mesh); r->adapt->mesh->acceptChanges(); } diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index 56d052340..63bed1064 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -322,9 +322,10 @@ void ZoltanData::setup() Zoltan_Set_Param(ztn, "obj_weight_dim", paramStr); Zoltan_Set_Param(ztn, "edge_weight_dim", "0"); + Mesh* m = zb->mesh; //Debug snprintf(paramStr, 128, "%d", dbgLvl); - if ( zb->isLocal && 0 != zb->mesh->getPCU()->Self() ) + if ( zb->isLocal && 0 != m->getPCU()->Self() ) snprintf(paramStr, 128, "%d", 0); //if local silence all but rank 0 Zoltan_Set_Param(ztn, "debug_level", paramStr); Zoltan_Set_Param(ztn, "PARMETIS_OUTPUT_LEVEL", paramStr); @@ -344,7 +345,7 @@ void ZoltanData::setup() if ( zb->isLocal ) { snprintf(paramStr, 128, "%d", zb->multiple); } else { - snprintf(paramStr, 128, "%d", zb->multiple*zb->mesh->getPCU()->Peers()); + snprintf(paramStr, 128, "%d", zb->multiple*m->getPCU()->Peers()); } Zoltan_Set_Param(ztn, "NUM_GLOBAL_PARTS", paramStr); snprintf(paramStr, 128, "%d", zb->multiple); From 7025e6214b0672a70b4a429c62fe21dc4cd1ebb2 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 19 Aug 2024 11:31:28 -0400 Subject: [PATCH 132/141] PCUObj renamed to PCU, PCU remaned to PCU_C --- apf/apfMesh.cc | 2 +- apf/apfMesh.h | 2 +- capstone_clis/capStone2VTK.cc | 2 +- capstone_clis/capStoneAnalyzeArea.cc | 2 +- capstone_clis/capStoneAnalyzeEdgeLengths.cc | 2 +- capstone_clis/capStoneAnisoAdapt.cc | 2 +- capstone_clis/capStoneAnisoAdaptWing.cc | 2 +- capstone_clis/capStoneAttachSolution.cc | 2 +- capstone_clis/capStoneAttachSolution2.cc | 2 +- .../capStoneAttachSolutionStrandOnly.cc | 2 +- capstone_clis/capStoneAttachSolutionUni.cc | 2 +- capstone_clis/capStoneCheckParametrization.cc | 2 +- capstone_clis/capStoneCurve.cc | 2 +- capstone_clis/capStoneGeomTest.cc | 2 +- capstone_clis/capStoneIsoAdaptB737.cc | 2 +- mds/apfMDS.cc | 2 +- mds/mdsCGNS.cc | 2 +- mds/mds_apf.c | 2 +- mds/mds_net.c | 2 +- mds/mds_order.c | 2 +- mds/mds_smb.c | 2 +- .../maximalIndependentSet/mersenne_twister.h | 2 +- parma/diffMC/maximalIndependentSet/mis.h | 2 +- parma/diffMC/parma_commons.h | 2 +- parma/diffMC/parma_stop.h | 2 +- pcu/CMakeLists.txt | 8 +- pcu/{PCUObj.cc => PCU.cc} | 2 +- pcu/PCU.h | 233 +++++++++--------- pcu/PCUObj.h | 131 ---------- pcu/PCU_C.h | 120 +++++++++ pcu/{pcu.cc => pcu_c.cc} | 2 +- pcu/pcu_io.c | 2 +- pcu/pkg_tribits.cmake | 6 +- phasta/phBC.cc | 1 - phasta/phCook.cc | 1 - phasta/phIO.c | 2 +- phasta/phiotimer.cc | 2 +- phasta/phstream.h | 2 +- phasta/readUrPrep.cc | 2 +- pumi/pumi_field.cc | 2 +- pumi/pumi_geom.cc | 2 +- pumi/pumi_mesh.cc | 2 +- python_wrappers/apf.i | 2 +- spr/spr.h | 2 +- test/assert_timing.cc | 2 +- test/capVol.cc | 2 +- test/cgns.cc | 2 +- test/generate.cc | 2 +- test/measureIsoStats.cc | 2 +- test/modelInfo.cc | 2 +- test/ph_adapt.cc | 2 +- test/pumi.cc | 2 +- test/repartition.cc | 2 +- test/sim_part.cc | 2 +- test/swapDoubles.cc | 3 +- 55 files changed, 297 insertions(+), 300 deletions(-) rename pcu/{PCUObj.cc => PCU.cc} (99%) delete mode 100644 pcu/PCUObj.h create mode 100644 pcu/PCU_C.h rename pcu/{pcu.cc => pcu_c.cc} (99%) diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 125dd825f..ee29e8125 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -5,7 +5,7 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include +#include #include "apfCoordData.h" #include "apfVectorField.h" #include "apfShape.h" diff --git a/apf/apfMesh.h b/apf/apfMesh.h index 5272263ec..c7b92b812 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "apfVector.h" #include "apfDynamicArray.h" diff --git a/capstone_clis/capStone2VTK.cc b/capstone_clis/capStone2VTK.cc index 8c741f784..8c4124cdc 100644 --- a/capstone_clis/capStone2VTK.cc +++ b/capstone_clis/capStone2VTK.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAnalyzeArea.cc b/capstone_clis/capStoneAnalyzeArea.cc index fdf4a0c90..86f83634b 100644 --- a/capstone_clis/capStoneAnalyzeArea.cc +++ b/capstone_clis/capStoneAnalyzeArea.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAnalyzeEdgeLengths.cc b/capstone_clis/capStoneAnalyzeEdgeLengths.cc index 31dd09403..ffce0c255 100644 --- a/capstone_clis/capStoneAnalyzeEdgeLengths.cc +++ b/capstone_clis/capStoneAnalyzeEdgeLengths.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAnisoAdapt.cc b/capstone_clis/capStoneAnisoAdapt.cc index c6d71cff9..1e0c4949c 100644 --- a/capstone_clis/capStoneAnisoAdapt.cc +++ b/capstone_clis/capStoneAnisoAdapt.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAnisoAdaptWing.cc b/capstone_clis/capStoneAnisoAdaptWing.cc index 5db6666be..39e7f53d4 100644 --- a/capstone_clis/capStoneAnisoAdaptWing.cc +++ b/capstone_clis/capStoneAnisoAdaptWing.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAttachSolution.cc b/capstone_clis/capStoneAttachSolution.cc index 0d8e7f647..2839c0772 100644 --- a/capstone_clis/capStoneAttachSolution.cc +++ b/capstone_clis/capStoneAttachSolution.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAttachSolution2.cc b/capstone_clis/capStoneAttachSolution2.cc index 797daecdd..1b156ffc7 100644 --- a/capstone_clis/capStoneAttachSolution2.cc +++ b/capstone_clis/capStoneAttachSolution2.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAttachSolutionStrandOnly.cc b/capstone_clis/capStoneAttachSolutionStrandOnly.cc index 8b2ffdae9..47d62d418 100644 --- a/capstone_clis/capStoneAttachSolutionStrandOnly.cc +++ b/capstone_clis/capStoneAttachSolutionStrandOnly.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneAttachSolutionUni.cc b/capstone_clis/capStoneAttachSolutionUni.cc index 416f1e41c..f781a970d 100644 --- a/capstone_clis/capStoneAttachSolutionUni.cc +++ b/capstone_clis/capStoneAttachSolutionUni.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneCheckParametrization.cc b/capstone_clis/capStoneCheckParametrization.cc index 704836d00..64bf03aba 100644 --- a/capstone_clis/capStoneCheckParametrization.cc +++ b/capstone_clis/capStoneCheckParametrization.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneCurve.cc b/capstone_clis/capStoneCurve.cc index 8d525da10..8f2da9ff8 100644 --- a/capstone_clis/capStoneCurve.cc +++ b/capstone_clis/capStoneCurve.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneGeomTest.cc b/capstone_clis/capStoneGeomTest.cc index dd2756134..fcc921c20 100644 --- a/capstone_clis/capStoneGeomTest.cc +++ b/capstone_clis/capStoneGeomTest.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/capstone_clis/capStoneIsoAdaptB737.cc b/capstone_clis/capStoneIsoAdaptB737.cc index e2cf07f00..819ce55d8 100644 --- a/capstone_clis/capStoneIsoAdaptB737.cc +++ b/capstone_clis/capStoneIsoAdaptB737.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 7ec95285e..65039867c 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -8,7 +8,7 @@ *******************************************************************************/ -#include +#include #include #include "apfMDS.h" #include "mds_apf.h" diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index fa2d7e883..24ad15f95 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -10,7 +10,7 @@ #include "apfShape.h" #include "gmi.h" /* this is for gmi_getline... */ #include -#include +#include #include #include #include "apfFieldData.h" diff --git a/mds/mds_apf.c b/mds/mds_apf.c index 92f6db313..13eb3a1cc 100644 --- a/mds/mds_apf.c +++ b/mds/mds_apf.c @@ -11,7 +11,7 @@ #include "mds_apf.h" #include #include -#include +#include struct mds_apf* mds_apf_create(struct gmi_model* model, int d, mds_id cap[MDS_TYPES]) diff --git a/mds/mds_net.c b/mds/mds_net.c index 1e8dd9ce5..ab59b86bb 100644 --- a/mds/mds_net.c +++ b/mds/mds_net.c @@ -9,7 +9,7 @@ *******************************************************************************/ #include "mds_net.h" -#include +#include #include #include #include diff --git a/mds/mds_order.c b/mds/mds_order.c index e02d51f07..ffc41b161 100644 --- a/mds/mds_order.c +++ b/mds/mds_order.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include struct queue { mds_id* e; diff --git a/mds/mds_smb.c b/mds/mds_smb.c index 3c96c2d07..882ebcc94 100644 --- a/mds/mds_smb.c +++ b/mds/mds_smb.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/parma/diffMC/maximalIndependentSet/mersenne_twister.h b/parma/diffMC/maximalIndependentSet/mersenne_twister.h index 66d82776c..a6dcbb365 100644 --- a/parma/diffMC/maximalIndependentSet/mersenne_twister.h +++ b/parma/diffMC/maximalIndependentSet/mersenne_twister.h @@ -1,7 +1,7 @@ #ifndef MERSENNE_TWISTER_H #define MERSENNE_TWISTER_H -#include +#include void mersenne_twister_seed(unsigned seed); unsigned mersenne_twister(pcu::PCU *PCUObj); diff --git a/parma/diffMC/maximalIndependentSet/mis.h b/parma/diffMC/maximalIndependentSet/mis.h index 23165fa83..291d7c6a4 100644 --- a/parma/diffMC/maximalIndependentSet/mis.h +++ b/parma/diffMC/maximalIndependentSet/mis.h @@ -10,7 +10,7 @@ #include "mpi.h" #include -#include "PCUObj.h" +#include "PCU.h" #define MIS_ITERATE(t,w,i) \ for (t::iterator i = (w).begin(); \ diff --git a/parma/diffMC/parma_commons.h b/parma/diffMC/parma_commons.h index 6e4e604ba..ab2fc588d 100644 --- a/parma/diffMC/parma_commons.h +++ b/parma/diffMC/parma_commons.h @@ -2,7 +2,7 @@ #define PARMA_COMMONS_H_ #include "apfArray.h" -#include "PCUObj.h" +#include "PCU.h" namespace parmaCommons { diff --git a/parma/diffMC/parma_stop.h b/parma/diffMC/parma_stop.h index 24439971e..ce7513126 100644 --- a/parma/diffMC/parma_stop.h +++ b/parma/diffMC/parma_stop.h @@ -2,7 +2,7 @@ #define PARMA_STOP_H #include "parma_monitor.h" -#include "PCUObj.h" +#include "PCU.h" namespace parma { class Stop { diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 927a52ef1..51ccbcdea 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,7 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.cc + pcu_c.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -22,16 +22,16 @@ set(SOURCES pcu_util.c noto/noto_malloc.c reel/reel.c - PCUObj.cc) + PCU.cc) # Package headers set(HEADERS - PCU.h + PCU_C.h pcu_io.h pcu_util.h reel/reel.h pcu_defines.h - PCUObj.h + PCU.h ) # Add the pcu library diff --git a/pcu/PCUObj.cc b/pcu/PCU.cc similarity index 99% rename from pcu/PCUObj.cc rename to pcu/PCU.cc index 9cf86af16..328628e51 100644 --- a/pcu/PCUObj.cc +++ b/pcu/PCU.cc @@ -1,4 +1,4 @@ -#include "PCUObj.h" +#include "PCU.h" #include "noto_malloc.h" #include "pcu_mem.h" #include "pcu_mpi.h" diff --git a/pcu/PCU.h b/pcu/PCU.h index a954832a9..40dc3ed39 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -1,120 +1,131 @@ -#ifndef PCU_H -#define PCU_H -#include "pcu_defines.h" -#include +#ifndef SCOREC_PCU_H +#define SCOREC_PCU_H -#ifdef __cplusplus -#include -#include -extern "C" { -#else -#include -#include -#include -#endif - -typedef struct PCU_t PCU_t; - -int PCU_Comm_Init(PCU_t* h); -int PCU_Comm_Free(PCU_t* h); - -/*rank/size functions*/ -int PCU_Comm_Self(PCU_t h); -int PCU_Comm_Peers(PCU_t h); - -/*recommended message passing API*/ -void PCU_Comm_Begin(PCU_t h); -int PCU_Comm_Pack(PCU_t h, int to_rank, const void* data, size_t size); -#define PCU_COMM_PACK(handle, to_rank,object)\ -PCU_Comm_Pack(handle, to_rank,&(object),sizeof(object)) -int PCU_Comm_Send(PCU_t h); -bool PCU_Comm_Receive(PCU_t h); -bool PCU_Comm_Listen(PCU_t h); -int PCU_Comm_Sender(PCU_t h); -bool PCU_Comm_Unpacked(PCU_t h); -int PCU_Comm_Unpack(PCU_t h, void* data, size_t size); -#define PCU_COMM_UNPACK(handle, object)\ -PCU_Comm_Unpack(handle, &(object),sizeof(object)) - -/*turns deterministic ordering for the - above API on/off*/ -void PCU_Comm_Order(PCU_t h, bool on); - -/*collective operations*/ -void PCU_Barrier(PCU_t h); -void PCU_Add_Doubles(PCU_t h, double* p, size_t n); -double PCU_Add_Double(PCU_t h, double x); -void PCU_Min_Doubles(PCU_t h, double* p, size_t n); -double PCU_Min_Double(PCU_t h, double x); -void PCU_Max_Doubles(PCU_t h, double* p, size_t n); -double PCU_Max_Double(PCU_t h, double x); -void PCU_Add_Ints(PCU_t h, int* p, size_t n); -int PCU_Add_Int(PCU_t h, int x); -void PCU_Add_Longs(PCU_t h, long* p, size_t n); -long PCU_Add_Long(PCU_t h, long x); -void PCU_Exscan_Ints(PCU_t h, int* p, size_t n); -int PCU_Exscan_Int(PCU_t h, int x); -void PCU_Exscan_Longs(PCU_t h, long* p, size_t n); -long PCU_Exscan_Long(PCU_t h, long x); -void PCU_Add_SizeTs(PCU_t h, size_t* p, size_t n); -size_t PCU_Add_SizeT(PCU_t h, size_t x); -void PCU_Min_SizeTs(PCU_t h, size_t* p, size_t n); -size_t PCU_Min_SizeT(PCU_t h, size_t x); -void PCU_Max_SizeTs(PCU_t h, size_t* p, size_t n); -size_t PCU_Max_SizeT(PCU_t h, size_t x); -void PCU_Min_Ints(PCU_t h, int* p, size_t n); -int PCU_Min_Int(PCU_t h, int x); -void PCU_Max_Ints(PCU_t h, int* p, size_t n); -int PCU_Max_Int(PCU_t h, int x); -void PCU_Max_Longs(PCU_t h, long* p, size_t n); -long PCU_Max_Long(PCU_t h, long x); -int PCU_Or(PCU_t h, int c); -int PCU_And(PCU_t h, int c); - -/*process-level self/peers (mpi wrappers)*/ -int PCU_Proc_Self(PCU_t h); -int PCU_Proc_Peers(PCU_t h); - -/*IPComMan replacement API*/ -int PCU_Comm_Write(PCU_t h, int to_rank, const void* data, size_t size); -#define PCU_COMM_WRITE(handle,to,data) \ -PCU_Comm_Write(handle, to,&(data),sizeof(data)) -bool PCU_Comm_Read(PCU_t h, int* from_rank, void** data, size_t* size); - -/*Debug file I/O API*/ -void PCU_Debug_Open(PCU_t h); - -void PCU_Debug_Print(PCU_t h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(2,3); -/*lesser-used APIs*/ -bool PCU_Comm_Initialized(PCU_t h); -int PCU_Comm_Packed(PCU_t h ,int to_rank, size_t* size); -int PCU_Comm_From(PCU_t h, int* from_rank); -int PCU_Comm_Received(PCU_t h, size_t* size); -void* PCU_Comm_Extract(PCU_t h, size_t size); -int PCU_Comm_Rank(PCU_t h, int* rank); -int PCU_Comm_Size(PCU_t h, int* size); - -/*deprecated method enum*/ - -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm(PCU_t h); +#include +#include +#include "pcu_defines.h" +struct pcu_msg_struct; +struct pcu_mpi_struct; + +namespace pcu { +class PCU { +public: + explicit PCU(MPI_Comm comm); + ~PCU() noexcept; + PCU(PCU const &) = delete; + PCU(PCU &&) noexcept; + PCU &operator=(PCU const &) = delete; + PCU &operator=(PCU &&) noexcept; + /** @brief Returns the rank of the current process. + * @return The rank of the current process. + */ + [[nodiscard]] int Self() const noexcept; + /** @brief Returns the number of ranks in the communicator. + * @return The number of ranks in the communicator. + */ + [[nodiscard]] int Peers() const noexcept; + [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; + + [[nodiscard]] PCU_t GetCHandle() {PCU_t h; h.ptr=this; return h;} + /*recommended message passing API*/ + void Begin() noexcept; + int Pack(int to_rank, const void *data, size_t size) noexcept; + template int Pack(int to_rank, T& data) noexcept { + return Pack(to_rank, &(data), sizeof(data)); + } + template int Pack(int to_rank, T*& data) noexcept { + return Pack(to_rank, &(data), sizeof(data)); + } + + int Send() noexcept; + bool Receive() noexcept; + bool Listen() noexcept; + int Sender() noexcept; + bool Unpacked() noexcept; + int Unpack(void *data, size_t size) noexcept; + template int Unpack(T& data) noexcept { + return Unpack(&(data), sizeof(data)); + } + template int Unpack(T*& data) noexcept { + return Unpack(&(data), sizeof(data)); + } + /*IPComMan replacement API*/ + int Write(int to_rank, const void *data, size_t size) noexcept; + bool Read(int *from_rank, void **data, size_t *size) noexcept; + + /*turns deterministic ordering for the + above API on/off*/ + void Order(bool on); + + /*collective operations*/ + void Barrier(); + template void Add(T *p, size_t n) noexcept; + template [[nodiscard]] T Add(T p) noexcept; + template void Min(T *p, size_t n) noexcept; + template [[nodiscard]] T Min(T p) noexcept; + template void Max(T *p, size_t n) noexcept; + template [[nodiscard]] T Max(T p) noexcept; + template void Exscan(T *p, size_t n) noexcept; + template [[nodiscard]] T Exscan(T p) noexcept; + + /*bitwise operations*/ + [[nodiscard]] int Or(int c) noexcept; + [[nodiscard]] int And(int c) noexcept; + + /*lesser-used APIs*/ + int Packed(int to_rank, size_t *size) noexcept; + int From(int *from_rank) noexcept; + int Received(size_t *size) noexcept; + void *Extract(size_t size) noexcept; + + void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3); + void DebugPrint(const char* format, va_list args) noexcept; + /* Debug functions */ + void DebugOpen() noexcept; + + MPI_Comm SwitchMPIComm(MPI_Comm) noexcept; + + //struct MPIComms { + // MPI_Comm original; + // MPI_Comm user; + // MPI_Comm coll; + //}; + // takes ownership of newcomms.user & newcomms.coll + // user responsibility to free returned user/coll comm + //MPIComms SwitchMPIComms(MPIComms& newcomms) noexcept; + +private: + pcu_msg_struct *msg_; + pcu_mpi_struct *mpi_; +}; /*stack trace helpers using GNU/Linux*/ -void PCU_Protect(void); - +void Protect() noexcept; +/*Memory usage*/ +[[nodiscard]] double GetMem() noexcept; /*MPI_Wtime() equivalent*/ -double PCU_Time(void); +[[nodiscard]] double Time() noexcept; -/*Memory usage*/ -double PCU_GetMem(void); +PCU* PCU_GetGlobal(); + +/* explicit instantiations of template functions */ +#define PCU_EXPL_INST_DECL(T) \ + extern template void PCU::Add(T * p, size_t n) noexcept; \ + extern template T PCU::Add(T p) noexcept; \ + extern template void PCU::Min(T * p, size_t n) noexcept; \ + extern template T PCU::Min(T p) noexcept; \ + extern template void PCU::Max(T * p, size_t n) noexcept; \ + extern template T PCU::Max(T p) noexcept; \ + extern template void PCU::Exscan(T * p, size_t n) noexcept; \ + extern template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL -/*Access global variable*/ -PCU_t PCU_Get_Global_Handle(void); +} // namespace pcu +#endif // PCUOBJ_H -#ifdef __cplusplus -} /* extern "C" */ -#endif -#endif \ No newline at end of file diff --git a/pcu/PCUObj.h b/pcu/PCUObj.h deleted file mode 100644 index 307e051b6..000000000 --- a/pcu/PCUObj.h +++ /dev/null @@ -1,131 +0,0 @@ -#ifndef SCOREC_PCU_PCUOBJ_H -#define SCOREC_PCU_PCUOBJ_H - -#include -#include -#include "pcu_defines.h" - -struct pcu_msg_struct; -struct pcu_mpi_struct; - -namespace pcu { -class PCU { -public: - explicit PCU(MPI_Comm comm); - ~PCU() noexcept; - PCU(PCU const &) = delete; - PCU(PCU &&) noexcept; - PCU &operator=(PCU const &) = delete; - PCU &operator=(PCU &&) noexcept; - /** @brief Returns the rank of the current process. - * @return The rank of the current process. - */ - [[nodiscard]] int Self() const noexcept; - /** @brief Returns the number of ranks in the communicator. - * @return The number of ranks in the communicator. - */ - [[nodiscard]] int Peers() const noexcept; - [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; - - [[nodiscard]] PCU_t GetCHandle() {PCU_t h; h.ptr=this; return h;} - /*recommended message passing API*/ - void Begin() noexcept; - int Pack(int to_rank, const void *data, size_t size) noexcept; - template int Pack(int to_rank, T& data) noexcept { - return Pack(to_rank, &(data), sizeof(data)); - } - template int Pack(int to_rank, T*& data) noexcept { - return Pack(to_rank, &(data), sizeof(data)); - } - - int Send() noexcept; - bool Receive() noexcept; - bool Listen() noexcept; - int Sender() noexcept; - bool Unpacked() noexcept; - int Unpack(void *data, size_t size) noexcept; - template int Unpack(T& data) noexcept { - return Unpack(&(data), sizeof(data)); - } - template int Unpack(T*& data) noexcept { - return Unpack(&(data), sizeof(data)); - } - /*IPComMan replacement API*/ - int Write(int to_rank, const void *data, size_t size) noexcept; - bool Read(int *from_rank, void **data, size_t *size) noexcept; - - /*turns deterministic ordering for the - above API on/off*/ - void Order(bool on); - - /*collective operations*/ - void Barrier(); - template void Add(T *p, size_t n) noexcept; - template [[nodiscard]] T Add(T p) noexcept; - template void Min(T *p, size_t n) noexcept; - template [[nodiscard]] T Min(T p) noexcept; - template void Max(T *p, size_t n) noexcept; - template [[nodiscard]] T Max(T p) noexcept; - template void Exscan(T *p, size_t n) noexcept; - template [[nodiscard]] T Exscan(T p) noexcept; - - /*bitwise operations*/ - [[nodiscard]] int Or(int c) noexcept; - [[nodiscard]] int And(int c) noexcept; - - /*lesser-used APIs*/ - int Packed(int to_rank, size_t *size) noexcept; - int From(int *from_rank) noexcept; - int Received(size_t *size) noexcept; - void *Extract(size_t size) noexcept; - - void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3); - void DebugPrint(const char* format, va_list args) noexcept; - /* Debug functions */ - void DebugOpen() noexcept; - - MPI_Comm SwitchMPIComm(MPI_Comm) noexcept; - - //struct MPIComms { - // MPI_Comm original; - // MPI_Comm user; - // MPI_Comm coll; - //}; - // takes ownership of newcomms.user & newcomms.coll - // user responsibility to free returned user/coll comm - //MPIComms SwitchMPIComms(MPIComms& newcomms) noexcept; - -private: - pcu_msg_struct *msg_; - pcu_mpi_struct *mpi_; -}; -/*stack trace helpers using GNU/Linux*/ -void Protect() noexcept; -/*Memory usage*/ -[[nodiscard]] double GetMem() noexcept; -/*MPI_Wtime() equivalent*/ -[[nodiscard]] double Time() noexcept; - -PCU* PCU_GetGlobal(); - -/* explicit instantiations of template functions */ -#define PCU_EXPL_INST_DECL(T) \ - extern template void PCU::Add(T * p, size_t n) noexcept; \ - extern template T PCU::Add(T p) noexcept; \ - extern template void PCU::Min(T * p, size_t n) noexcept; \ - extern template T PCU::Min(T p) noexcept; \ - extern template void PCU::Max(T * p, size_t n) noexcept; \ - extern template T PCU::Max(T p) noexcept; \ - extern template void PCU::Exscan(T * p, size_t n) noexcept; \ - extern template T PCU::Exscan(T p) noexcept; -PCU_EXPL_INST_DECL(int) -PCU_EXPL_INST_DECL(size_t) -PCU_EXPL_INST_DECL(long) -PCU_EXPL_INST_DECL(double) -#undef PCU_EXPL_INST_DECL - -} // namespace pcu - -#endif // PCUOBJ_H - - diff --git a/pcu/PCU_C.h b/pcu/PCU_C.h new file mode 100644 index 000000000..e8a05169c --- /dev/null +++ b/pcu/PCU_C.h @@ -0,0 +1,120 @@ +#ifndef PCU_C_H +#define PCU_C_H +#include "pcu_defines.h" +#include + +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif + +typedef struct PCU_t PCU_t; + +int PCU_Comm_Init(PCU_t* h); +int PCU_Comm_Free(PCU_t* h); + +/*rank/size functions*/ +int PCU_Comm_Self(PCU_t h); +int PCU_Comm_Peers(PCU_t h); + +/*recommended message passing API*/ +void PCU_Comm_Begin(PCU_t h); +int PCU_Comm_Pack(PCU_t h, int to_rank, const void* data, size_t size); +#define PCU_COMM_PACK(handle, to_rank,object)\ +PCU_Comm_Pack(handle, to_rank,&(object),sizeof(object)) +int PCU_Comm_Send(PCU_t h); +bool PCU_Comm_Receive(PCU_t h); +bool PCU_Comm_Listen(PCU_t h); +int PCU_Comm_Sender(PCU_t h); +bool PCU_Comm_Unpacked(PCU_t h); +int PCU_Comm_Unpack(PCU_t h, void* data, size_t size); +#define PCU_COMM_UNPACK(handle, object)\ +PCU_Comm_Unpack(handle, &(object),sizeof(object)) + +/*turns deterministic ordering for the + above API on/off*/ +void PCU_Comm_Order(PCU_t h, bool on); + +/*collective operations*/ +void PCU_Barrier(PCU_t h); +void PCU_Add_Doubles(PCU_t h, double* p, size_t n); +double PCU_Add_Double(PCU_t h, double x); +void PCU_Min_Doubles(PCU_t h, double* p, size_t n); +double PCU_Min_Double(PCU_t h, double x); +void PCU_Max_Doubles(PCU_t h, double* p, size_t n); +double PCU_Max_Double(PCU_t h, double x); +void PCU_Add_Ints(PCU_t h, int* p, size_t n); +int PCU_Add_Int(PCU_t h, int x); +void PCU_Add_Longs(PCU_t h, long* p, size_t n); +long PCU_Add_Long(PCU_t h, long x); +void PCU_Exscan_Ints(PCU_t h, int* p, size_t n); +int PCU_Exscan_Int(PCU_t h, int x); +void PCU_Exscan_Longs(PCU_t h, long* p, size_t n); +long PCU_Exscan_Long(PCU_t h, long x); +void PCU_Add_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Add_SizeT(PCU_t h, size_t x); +void PCU_Min_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Min_SizeT(PCU_t h, size_t x); +void PCU_Max_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Max_SizeT(PCU_t h, size_t x); +void PCU_Min_Ints(PCU_t h, int* p, size_t n); +int PCU_Min_Int(PCU_t h, int x); +void PCU_Max_Ints(PCU_t h, int* p, size_t n); +int PCU_Max_Int(PCU_t h, int x); +void PCU_Max_Longs(PCU_t h, long* p, size_t n); +long PCU_Max_Long(PCU_t h, long x); +int PCU_Or(PCU_t h, int c); +int PCU_And(PCU_t h, int c); + +/*process-level self/peers (mpi wrappers)*/ +int PCU_Proc_Self(PCU_t h); +int PCU_Proc_Peers(PCU_t h); + +/*IPComMan replacement API*/ +int PCU_Comm_Write(PCU_t h, int to_rank, const void* data, size_t size); +#define PCU_COMM_WRITE(handle,to,data) \ +PCU_Comm_Write(handle, to,&(data),sizeof(data)) +bool PCU_Comm_Read(PCU_t h, int* from_rank, void** data, size_t* size); + +/*Debug file I/O API*/ +void PCU_Debug_Open(PCU_t h); + +void PCU_Debug_Print(PCU_t h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(2,3); +/*lesser-used APIs*/ +bool PCU_Comm_Initialized(PCU_t h); +int PCU_Comm_Packed(PCU_t h ,int to_rank, size_t* size); +int PCU_Comm_From(PCU_t h, int* from_rank); +int PCU_Comm_Received(PCU_t h, size_t* size); +void* PCU_Comm_Extract(PCU_t h, size_t size); +int PCU_Comm_Rank(PCU_t h, int* rank); +int PCU_Comm_Size(PCU_t h, int* size); + +/*deprecated method enum*/ + +/*special MPI_Comm replacement API*/ +void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm); +MPI_Comm PCU_Get_Comm(PCU_t h); + +/*stack trace helpers using GNU/Linux*/ +void PCU_Protect(void); + +/*MPI_Wtime() equivalent*/ +double PCU_Time(void); + +/*Memory usage*/ +double PCU_GetMem(void); + +/*Access global variable*/ +PCU_t PCU_Get_Global_Handle(void); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif \ No newline at end of file diff --git a/pcu/pcu.cc b/pcu/pcu_c.cc similarity index 99% rename from pcu/pcu.cc rename to pcu/pcu_c.cc index e1f1a7feb..c763cb19f 100644 --- a/pcu/pcu.cc +++ b/pcu/pcu_c.cc @@ -1,5 +1,5 @@ +#include "PCU_C.h" #include "PCU.h" -#include "PCUObj.h" #include "reel.h" #include diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index ec6eb21f4..5d698984e 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,7 +8,7 @@ *******************************************************************************/ #include "pcu_io.h" -#include "PCU.h" +#include "PCU_C.h" #include "noto_malloc.h" #include "pcu_buffer.h" #include "pcu_util.h" diff --git a/pcu/pkg_tribits.cmake b/pcu/pkg_tribits.cmake index ce6e88e92..ec9a43a16 100644 --- a/pcu/pkg_tribits.cmake +++ b/pcu/pkg_tribits.cmake @@ -33,7 +33,7 @@ include_directories("${PROJECT_BINARY_DIR}") #Sources & Headers set(SOURCES - pcu.cc + pcu_c.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -46,10 +46,10 @@ set(SOURCES pcu_util.c noto/noto_malloc.c reel/reel.c - PCUObj.cc) + PCU.cc) set(HEADERS - PCU.h + PCU_C.h pcu_io.h pcu_util.h reel/reel.h diff --git a/phasta/phBC.cc b/phasta/phBC.cc index 00c39d2d3..f50836170 100644 --- a/phasta/phBC.cc +++ b/phasta/phBC.cc @@ -1,4 +1,3 @@ -//#include #include #include "phBC.h" #ifdef HAVE_SIMMETRIX diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 07a5ebb05..670473a53 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -22,7 +22,6 @@ #include #include #include -//#include #include #include #include diff --git a/phasta/phIO.c b/phasta/phIO.c index d94caf120..caf3c590a 100644 --- a/phasta/phIO.c +++ b/phasta/phIO.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index 2f53909d6..b4b4ffc0d 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -1,7 +1,7 @@ #include #include +#include #include -#include #include #include #include diff --git a/phasta/phstream.h b/phasta/phstream.h index e1747f180..316e074c5 100644 --- a/phasta/phstream.h +++ b/phasta/phstream.h @@ -1,7 +1,7 @@ #ifndef PHSTREAM_H_ #define PHSTREAM_H_ #include -#include +#include /** \file phstream.h diff --git a/phasta/readUrPrep.cc b/phasta/readUrPrep.cc index 9e762e5be..d6ea4e1e5 100644 --- a/phasta/readUrPrep.cc +++ b/phasta/readUrPrep.cc @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include diff --git a/pumi/pumi_field.cc b/pumi/pumi_field.cc index 61e0ed14d..9f4707c8e 100644 --- a/pumi/pumi_field.cc +++ b/pumi/pumi_field.cc @@ -13,7 +13,7 @@ #include "apfFieldData.h" #include "apfNumbering.h" #include -#include +#include #include #include // for malloc and free diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index 47b407bf0..2305959b1 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -12,7 +12,7 @@ #include "gmi_null.h" #include "gmi_analytic.h" #include "pumi_iter.h" -#include +#include #include #include #include diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index b20051b40..09b4c3745 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i index 68cc8372e..283b32005 100644 --- a/python_wrappers/apf.i +++ b/python_wrappers/apf.i @@ -2,7 +2,7 @@ %{ #include #include -#include +#include #include #include #include diff --git a/spr/spr.h b/spr/spr.h index cd5a2007f..d70978e3b 100644 --- a/spr/spr.h +++ b/spr/spr.h @@ -16,7 +16,7 @@ #include "apfNew.h" #include "apfDynamicVector.h" #include "apfDynamicMatrix.h" -#include "PCUObj.h" +#include "PCU.h" /** \namespace spr diff --git a/test/assert_timing.cc b/test/assert_timing.cc index f65dcd52c..63e679263 100644 --- a/test/assert_timing.cc +++ b/test/assert_timing.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/test/capVol.cc b/test/capVol.cc index 9f9c0b8c5..24c99483d 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -5,7 +5,7 @@ #include // Parallelism -#include +#include #include // Mesh interfaces diff --git a/test/cgns.cc b/test/cgns.cc index 316beb927..d8713cc8c 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include // diff --git a/test/generate.cc b/test/generate.cc index a34f0f6cc..1a8b31ffb 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/test/measureIsoStats.cc b/test/measureIsoStats.cc index 30bab544a..87b60f760 100644 --- a/test/measureIsoStats.cc +++ b/test/measureIsoStats.cc @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include // === includes for safe_mkdir === diff --git a/test/modelInfo.cc b/test/modelInfo.cc index b11737703..2fa9865b7 100644 --- a/test/modelInfo.cc +++ b/test/modelInfo.cc @@ -9,7 +9,7 @@ #include #endif #include //exit and exit_failure -#include +#include #include #include diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index de9226922..7593ddcb5 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/test/pumi.cc b/test/pumi.cc index 320254d99..48adfb108 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/test/repartition.cc b/test/repartition.cc index 98caca57e..08bf1c3bb 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/test/sim_part.cc b/test/sim_part.cc index 16c96ef90..c43b8fbee 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -39,7 +39,7 @@ #endif #endif -#include +#include #include #include #include diff --git a/test/swapDoubles.cc b/test/swapDoubles.cc index 7db5e5966..6019ef4fc 100644 --- a/test/swapDoubles.cc +++ b/test/swapDoubles.cc @@ -1,5 +1,4 @@ -//#include -#include +#include #include //pcu_swap_doubles #include //PCU_ALWAYS_ASSERT #include //iota From 90785edb82d4310f63e5ddaef9521c56824d4edf Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 19 Aug 2024 16:39:13 -0400 Subject: [PATCH 133/141] Specified argument data type for templated functions PCU:: Add, Max, and Min --- apf/apfCGNS.cc | 8 +-- apf/apfCavityOp.cc | 2 +- apf/apfConstruct.cc | 2 +- apf/apfConvertTags.h | 2 +- apf/apfMesh.cc | 6 +-- apf/apfVerify.cc | 6 +-- crv/crvAdapt.cc | 2 +- crv/crvShape.cc | 8 +-- ma/maAdapt.cc | 2 +- ma/maCoarsen.cc | 2 +- ma/maLayer.cc | 4 +- ma/maLayerCoarsen.cc | 4 +- ma/maLayerSnap.cc | 6 +-- ma/maMatch.cc | 2 +- ma/maMesh.cc | 10 ++-- ma/maShape.cc | 2 +- ma/maSize.cc | 4 +- ma/maSnap.cc | 6 +-- ma/maTetrahedronize.cc | 2 +- parma/diffMC/maximalIndependentSet/misLuby.cc | 2 +- .../maximalIndependentSet/test/testMIS.cc | 2 +- parma/diffMC/parma_commons.cc | 2 +- parma/diffMC/parma_dcpartFixer.cc | 2 +- parma/diffMC/parma_entWeights.cc | 10 ++-- parma/diffMC/parma_graphDist.cc | 6 +-- parma/diffMC/parma_shapeOptimizer.cc | 4 +- parma/diffMC/parma_shapeTargets.cc | 2 +- parma/diffMC/parma_sides.cc | 2 +- parma/diffMC/parma_step.cc | 2 +- parma/diffMC/parma_vtxPtnWriter.cc | 2 +- parma/parma.cc | 50 +++++++++---------- phasta/phAdapt.cc | 2 +- phasta/phCook.cc | 6 +-- phasta/phGrowthCurves.cc | 4 +- phasta/phInterfaceCutter.cc | 2 +- phasta/phOutput.cc | 16 +++--- phasta/phPartition.cc | 2 +- phasta/ph_convert.cc | 4 +- pumi/pumi_field.cc | 2 +- sam/samElementCount.cc | 2 +- spr/sprEstimateError.cc | 2 +- spr/sprEstimateTargetError.cc | 2 +- test/curvetest.cc | 2 +- test/describe.cc | 6 +-- test/matchedNodeElmReader.cc | 2 +- test/quality.cc | 10 ++-- test/visualizeAnisoSizes.cc | 2 +- 47 files changed, 116 insertions(+), 116 deletions(-) diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 33037d47c..71695d719 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -45,7 +45,7 @@ static Count count(apf::Mesh *m, int dim) { const int local = apf::countOwned(m, dim); int total = local; - m->getPCU()->Add(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array return std::make_pair(total, local); } @@ -288,7 +288,7 @@ void WriteFields(const CGNS &cgns, const std::vectorgetPCU()->Add(&size, 1); // size of total array + m->getPCU()->Add(&size, 1); // size of total array // oddness of the api rmin[1] = rmin[0]; @@ -503,7 +503,7 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe m->end(cellIter); numbersByElementType[o] = counter; int total = counter; - m->getPCU()->Add(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array globalNumbersByElementType[o] = total; } cgsize_t allTotal = std::accumulate(globalNumbersByElementType.begin(), globalNumbersByElementType.end(), 0); @@ -633,7 +633,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, int startOfBCBlock = startingLocation + 1; const int number = bc.second.size(); int total = number; - m->getPCU()->Add(&total, 1); // size of total array + m->getPCU()->Add 0) { const auto allEnd = startOfBCBlock + total - 1; //one-based diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 4d9097116..146f0a084 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -133,7 +133,7 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = mesh->getPCU()->Min(static_cast(requests.empty())); + int done = mesh->getPCU()->Min(static_cast(requests.empty())); if (done) return false; /* throw in the local pull requests */ int self = mesh->getPCU()->Self(); diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index 218539c15..8c2a25c61 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -47,7 +47,7 @@ static Gid getMax(const GlobalToVert& globalToVert, Mesh2* m) Gid max = -1; APF_CONST_ITERATE(GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return m->getPCU()->Max(max); // this is type-dependent + return m->getPCU()->Max(max); // this is type-dependent } diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h index 4905cdccf..e4dbd446b 100644 --- a/apf/apfConvertTags.h +++ b/apf/apfConvertTags.h @@ -10,7 +10,7 @@ namespace { apf::Gid max = -1; APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return pcu_obj->Max(max); // this is type-dependent + return pcu_obj->Max(max); // this is type-dependent } template inline diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index ee29e8125..7f57fc983 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -872,7 +872,7 @@ void printTypes(Mesh* m) while ((e = m->iterate(it))) typeCnt[m->getType(e)]++; m->end(it); - m->getPCU()->Add(typeCnt,Mesh::TYPES); + m->getPCU()->Add(typeCnt,Mesh::TYPES); if (!m->getPCU()->Self()) { lion_oprint(1,"number of"); for (int i=0; igetPCU()->Add(n, 4); + m->getPCU()->Add(n, 4); printTypes(m); if (!m->getPCU()->Self()) lion_oprint(1,"mesh entity counts: v %ld e %ld f %ld r %ld\n", @@ -899,7 +899,7 @@ void warnAboutEmptyParts(Mesh* m) int emptyParts = 0; if (!m->count(m->getDimension())) ++emptyParts; - emptyParts = m->getPCU()->Add(emptyParts); + emptyParts = m->getPCU()->Add(emptyParts); if (emptyParts && (!m->getPCU()->Self())) lion_eprint(1,"APF warning: %d empty parts\n",emptyParts); } diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc index 6b2969157..a348fa40a 100644 --- a/apf/apfVerify.cc +++ b/apf/apfVerify.cc @@ -456,7 +456,7 @@ static long verifyCoords(Mesh* m) while (m->getPCU()->Receive()) if (!receiveCoords(m)) ++n; - return m->getPCU()->Add(n); + return m->getPCU()->Add(n); } long verifyVolumes(Mesh* m, bool printVolumes) @@ -482,7 +482,7 @@ long verifyVolumes(Mesh* m, bool printVolumes) } } m->end(it); - return m->getPCU()->Add(n); + return m->getPCU()->Add(n); } static void packAlignment(Mesh* m, MeshEntity* e, MeshEntity* r, int to) @@ -722,7 +722,7 @@ static void receiveTagData(Mesh* m, DynamicArray& tags) } // switch } // while - int global_size = m->getPCU()->Max((int)mismatch_tags.size()); + int global_size = m->getPCU()->Max((int)mismatch_tags.size()); if (global_size&&!m->getPCU()->Self()) for (std::set::iterator it=mismatch_tags.begin(); it!=mismatch_tags.end(); ++it) lion_oprint(1," - tag \"%s\" data mismatch over remote/ghost copies\n", m->getTagName(*it)); diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc index edc2ff3d3..6d2ab52b4 100644 --- a/crv/crvAdapt.cc +++ b/crv/crvAdapt.cc @@ -131,7 +131,7 @@ int markInvalidEntities(Adapt* a) } m->end(it); delete qual; - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } int getTag(Adapt* a, ma::Entity* e) diff --git a/crv/crvShape.cc b/crv/crvShape.cc index be35ea8da..2001671cb 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -634,7 +634,7 @@ static int markEdgesOppLargeAnglesTri(Adapt* a) } m->end(it); } while(count > prev_count); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } static int markEdgesOppLargeAnglesTet(Adapt* a) @@ -662,7 +662,7 @@ static int markEdgesOppLargeAnglesTet(Adapt* a) } m->end(it); } while(count > prev_count); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } /* The whole idea is to do the quality check once, @@ -700,7 +700,7 @@ static int markEdgesToFix(Adapt* a, int flag) } m->end(it); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } int fixLargeBoundaryAngles(Adapt* a) @@ -732,7 +732,7 @@ static void collapseInvalidEdges(Adapt* a) findIndependentSet(a); successCount += ma::collapseAllEdges(a, modelDimension); } - successCount = m->getPCU()->Add(successCount); + successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); ma::print(m->getPCU(), "Collapsed %d bad edges " "in %f seconds", successCount, t1-t0); diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index b99bc7282..bff23cf90 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -320,7 +320,7 @@ long markEntities( setFlag(a,e,setFalseFlag); } m->end(it); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } void NewEntities::reset() diff --git a/ma/maCoarsen.cc b/ma/maCoarsen.cc index 7407f2493..35089e344 100644 --- a/ma/maCoarsen.cc +++ b/ma/maCoarsen.cc @@ -247,7 +247,7 @@ bool coarsen(Adapt* a) else successCount += collapseAllEdges(a, modelDimension); } - successCount = m->getPCU()->Add(successCount); + successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); print(m->getPCU(), "coarsened %li edges in %f seconds", successCount,t1-t0); return true; diff --git a/ma/maLayer.cc b/ma/maLayer.cc index 0f419ca2f..e53cd814c 100644 --- a/ma/maLayer.cc +++ b/ma/maLayer.cc @@ -37,7 +37,7 @@ static long markLayerElements(Adapt* a) } } } - n = m->getPCU()->Add(n); + n = m->getPCU()->Add(n); a->hasLayer = (n != 0); if ( ! a->hasLayer) return 0; @@ -186,7 +186,7 @@ void checkLayerShape(Mesh* m, const char* key) ++n; } m->end(it); - n = m->getPCU()->Add(n); + n = m->getPCU()->Add(n); double t1 = pcu::Time(); print(m->getPCU(), "%s: checked layer quality in %f seconds: %ld unsafe elements", key, t1 - t0, n); } diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index 150a0101c..0fe7670ca 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -40,7 +40,7 @@ static long markBaseEdgesToCollapse(Adapt* a) } } m->end(it); - return m->getPCU()->Add(n); + return m->getPCU()->Add(n); } struct CurveLocalizer : public Crawler @@ -258,7 +258,7 @@ static long collapseAllStacks(Adapt* a, int d) allSuccesses += successCount; ++round; } while (a->mesh->getPCU()->Or(skipCount)); - return a->mesh->getPCU()->Add(allSuccesses); + return a->mesh->getPCU()->Add(allSuccesses); } bool coarsenLayer(Adapt* a) diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index 63ecd4b80..23b1bb1dc 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -233,7 +233,7 @@ struct LayerSnapper : public Crawler } } syncLayer(this, owned); - m->getPCU()->Add(&ncurves, 1); + m->getPCU()->Add(&ncurves, 1); } void end() { @@ -436,7 +436,7 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) setFlag(a, v, LAYER_UNSNAP); PCU_ALWAYS_ASSERT(m->hasTag(v, snapTag)); } - n = m->getPCU()->Add(n); + n = m->getPCU()->Add(n); print(m->getPCU(), "fed back unsnap flag from %ld tops", n); } @@ -485,7 +485,7 @@ struct Unsnapper : public Crawler ++ncurves; } } - m->getPCU()->Add(&ncurves, 1); + m->getPCU()->Add(&ncurves, 1); syncLayer(this, owned); } void end() diff --git a/ma/maMatch.cc b/ma/maMatch.cc index 6cc1c9474..c17ecdda5 100644 --- a/ma/maMatch.cc +++ b/ma/maMatch.cc @@ -71,7 +71,7 @@ void matchNewElements(Refine* r) } } } - face_count = m->getPCU()->Add(face_count); + face_count = m->getPCU()->Add(face_count); print(m->getPCU(), "updated matching for %li faces", face_count); } diff --git a/ma/maMesh.cc b/ma/maMesh.cc index c27c1d3b1..3e0f370e8 100644 --- a/ma/maMesh.cc +++ b/ma/maMesh.cc @@ -279,8 +279,8 @@ void getBoundingBox(Mesh* m, Vector& lower, Vector& upper) lower.toArray(a); double b[3]; upper.toArray(b); - m->getPCU()->Min(a, 3); - m->getPCU()->Max(b, 3); + m->getPCU()->Min(a, 3); + m->getPCU()->Max(b, 3); lower.fromArray(a); upper.fromArray(b); } @@ -302,7 +302,7 @@ Vector getCentroid(Mesh* m) } m->end(it); pointSum.toArray(values); - m->getPCU()->Add(&(values[0]),4); + m->getPCU()->Add(&(values[0]),4); return Vector(values)/values[3]; } @@ -369,7 +369,7 @@ double getAverageElementSize(Mesh* m) m->end(it); double& count = sums[1]; count = m->count(m->getDimension()); - m->getPCU()->Add(sums,2); + m->getPCU()->Add(sums,2); return sizeSum / count; } @@ -384,7 +384,7 @@ double getMinimumElementSize(Mesh* m) if (size < minimum) minimum=size; } m->end(it); - return m->getPCU()->Min(minimum); + return m->getPCU()->Min(minimum); } void getFaceEdgesAndDirections( diff --git a/ma/maShape.cc b/ma/maShape.cc index 67812eead..f34ebcc29 100644 --- a/ma/maShape.cc +++ b/ma/maShape.cc @@ -165,7 +165,7 @@ double getMinQuality(Adapt* a) minqual = qual; } m->end(it); - return m->getPCU()->Min(minqual); + return m->getPCU()->Min(minqual); } class ShortEdgeFixer : public Operator diff --git a/ma/maSize.cc b/ma/maSize.cc index 2a403dfc1..7f097d2eb 100644 --- a/ma/maSize.cc +++ b/ma/maSize.cc @@ -666,7 +666,7 @@ double getAverageEdgeLength(Mesh* m) edge_count += 1.0; } m->end(it); - m->getPCU()->Add(sums,2); + m->getPCU()->Add(sums,2); return length_sum / edge_count; } @@ -686,7 +686,7 @@ double getMaximumEdgeLength(Mesh* m, SizeField* sf) maxLength = length; } m->end(it); - m->getPCU()->Max(&maxLength,1); + m->getPCU()->Max(&maxLength,1); return maxLength; } diff --git a/ma/maSnap.cc b/ma/maSnap.cc index feb55ab43..40bb54d75 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -779,7 +779,7 @@ bool snapAllVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) { SnapAll op(a, t, isSimple); applyOperator(a, &op); - successCount += a->mesh->getPCU()->Add(op.successCount); + successCount += a->mesh->getPCU()->Add(op.successCount); return a->mesh->getPCU()->Or(op.didAnything); } @@ -830,7 +830,7 @@ bool snapMatchedVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) { SnapMatched op(a, t, isSimple); applyOperator(a, &op); - successCount += a->mesh->getPCU()->Add(op.successCount); + successCount += a->mesh->getPCU()->Add(op.successCount); return a->mesh->getPCU()->Or(op.didAnything); } @@ -856,7 +856,7 @@ long tagVertsToSnap(Adapt* a, Tag*& t) ++n; } m->end(it); - return m->getPCU()->Add(n); + return m->getPCU()->Add(n); } static void markVertsToSnap(Adapt* a, Tag* t) diff --git a/ma/maTetrahedronize.cc b/ma/maTetrahedronize.cc index fd2cb1e11..36b62f64f 100644 --- a/ma/maTetrahedronize.cc +++ b/ma/maTetrahedronize.cc @@ -619,7 +619,7 @@ static long markIslandPyramids(Adapt* a) } } m->end(it); - return m->getPCU()->Add(n); + return m->getPCU()->Add(n); } static int countEntitiesWithFlag(Adapt* a, int flag, int dim) diff --git a/parma/diffMC/maximalIndependentSet/misLuby.cc b/parma/diffMC/maximalIndependentSet/misLuby.cc index 896f19c6e..ecabc915a 100644 --- a/parma/diffMC/maximalIndependentSet/misLuby.cc +++ b/parma/diffMC/maximalIndependentSet/misLuby.cc @@ -459,7 +459,7 @@ int mis(partInfo& part, pcu::PCU *PCUObj, bool randNumsPredefined, bool isNeighb nodesRemoved.clear(); rmtNodesToRemove.clear(); - numNodesAdded = PCUObj->Add(numNodesAdded); + numNodesAdded = PCUObj->Add(numNodesAdded); } while (numNodesAdded > 0); return isInMis; diff --git a/parma/diffMC/maximalIndependentSet/test/testMIS.cc b/parma/diffMC/maximalIndependentSet/test/testMIS.cc index f12599d65..5fc93af91 100644 --- a/parma/diffMC/maximalIndependentSet/test/testMIS.cc +++ b/parma/diffMC/maximalIndependentSet/test/testMIS.cc @@ -302,7 +302,7 @@ int test_2dStencil(const int rank, const int totNumParts, pcu::PCU *PCUObj const double t1 = pcu::Time(); int isInMis = mis(part, PCUObj, randNumsPredefined,isNeighbors); double elapsedTime = pcu::Time() - t1; - PCUObj->Max(&elapsedTime, 1); + PCUObj->Max(&elapsedTime, 1); if( !PCUObj->Self() ) status("elapsed time (seconds) = %f \n", elapsedTime); diff --git a/parma/diffMC/parma_commons.cc b/parma/diffMC/parma_commons.cc index cfd46df55..80b228ab7 100644 --- a/parma/diffMC/parma_commons.cc +++ b/parma/diffMC/parma_commons.cc @@ -45,7 +45,7 @@ int parmaCommons::isMore(double a, double b) { } void parmaCommons::printElapsedTime(const char* fn, double elapsed, pcu::PCU *PCUObj) { - elapsed = PCUObj->Max(elapsed); + elapsed = PCUObj->Max(elapsed); if( !PCUObj->Self() ) status("%s elapsed time %lf seconds\n", fn, elapsed); } diff --git a/parma/diffMC/parma_dcpartFixer.cc b/parma/diffMC/parma_dcpartFixer.cc index 6bb88f1c4..9576e7eed 100644 --- a/parma/diffMC/parma_dcpartFixer.cc +++ b/parma/diffMC/parma_dcpartFixer.cc @@ -40,7 +40,7 @@ class dcPartFixer::PartFixer : public dcPart { int totNumDc() { int ndc = TO_INT(numDisconnectedComps()); - return m->getPCU()->Add(ndc); + return m->getPCU()->Add(ndc); } void setupPlan(muu& dcCompTgts, apf::Migration* plan) { diff --git a/parma/diffMC/parma_entWeights.cc b/parma/diffMC/parma_entWeights.cc index ed1f062b5..ca085593d 100644 --- a/parma/diffMC/parma_entWeights.cc +++ b/parma/diffMC/parma_entWeights.cc @@ -5,12 +5,12 @@ namespace parma { double getMaxWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return m->getPCU()->Max(locW); + return m->getPCU()->Max(locW); } double getAvgWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return m->getPCU()->Add(locW) / m->getPCU()->Peers(); + return m->getPCU()->Add(locW) / m->getPCU()->Peers(); } double getWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { @@ -27,14 +27,14 @@ namespace parma { void getImbalance(Weights* w, double& imb, double& avg, pcu::PCU *PCUObj) { double sum, max; sum = max = w->self(); - sum = PCUObj->Add(sum); - max = PCUObj->Max(max); + sum = PCUObj->Add(sum); + max = PCUObj->Max(max); avg = sum/PCUObj->Peers(); imb = max/avg; } double getMaxWeight(Weights* w, pcu::PCU *PCUObj) { - return PCUObj->Max(w->self()); + return PCUObj->Max(w->self()); } double getEntWeight(apf::Mesh* m, apf::MeshEntity* e, apf::MeshTag* w) { diff --git a/parma/diffMC/parma_graphDist.cc b/parma/diffMC/parma_graphDist.cc index 7cd60570e..d9114257e 100644 --- a/parma/diffMC/parma_graphDist.cc +++ b/parma/diffMC/parma_graphDist.cc @@ -342,9 +342,9 @@ namespace parma_ordering { } m->end(it); m->getPCU()->DebugPrint("la %d\n", la); - long tot=m->getPCU()->Add(TO_LONG(la)); - int max=m->getPCU()->Max(la); - int min=m->getPCU()->Min(la); + long tot=m->getPCU()->Add(TO_LONG(la)); + int max=m->getPCU()->Max(la); + int min=m->getPCU()->Min(la); double avg = TO_DOUBLE(tot)/m->getPCU()->Peers(); if( !m->getPCU()->Self() ) parmaCommons::status("la min %d max %d avg %.3f\n", min, max, avg); diff --git a/parma/diffMC/parma_shapeOptimizer.cc b/parma/diffMC/parma_shapeOptimizer.cc index 690f2c5d7..15eb8d8d6 100644 --- a/parma/diffMC/parma_shapeOptimizer.cc +++ b/parma/diffMC/parma_shapeOptimizer.cc @@ -14,7 +14,7 @@ namespace { using parmaCommons::status; int getMaxNb(parma::Sides* s, pcu::PCU *PCUObj) { - return PCUObj->Max(s->size()); + return PCUObj->Max(s->size()); } class ImbOrMaxNeighbor : public parma::Stop { @@ -22,7 +22,7 @@ namespace { ImbOrMaxNeighbor(parma::Average* nbAvg, double maxNbTol, int targets, int v=0) : nb(nbAvg), nbTol(maxNbTol), tgts(targets), verbose(v) {} bool stop(double imb, double maxImb, pcu::PCU *PCUObj) { - int maxTgts = PCUObj->Max(tgts); + int maxTgts = PCUObj->Max(tgts); const double nbSlope = nb->avg(); if( !PCUObj->Self() && verbose ) status("max neighbor slope %f tolerance %f\n", nbSlope, nbTol); diff --git a/parma/diffMC/parma_shapeTargets.cc b/parma/diffMC/parma_shapeTargets.cc index 80d466631..7fd3baa1c 100644 --- a/parma/diffMC/parma_shapeTargets.cc +++ b/parma/diffMC/parma_shapeTargets.cc @@ -26,7 +26,7 @@ namespace parma { int smallLimit; double totW; void init(Sides* s, pcu::PCU *PCUObj) { - const unsigned maxNb = TO_UINT(PCUObj->Max(s->size())); + const unsigned maxNb = TO_UINT(PCUObj->Max(s->size())); if( s->size() != maxNb ) return; PCUObj->DebugPrint("maxNb %d\n", maxNb); std::string sstr = s->print("sides"); diff --git a/parma/diffMC/parma_sides.cc b/parma/diffMC/parma_sides.cc index 3febcbe23..62d6b8bf0 100644 --- a/parma/diffMC/parma_sides.cc +++ b/parma/diffMC/parma_sides.cc @@ -2,6 +2,6 @@ #include "parma_convert.h" double parma::avgSharedSides(parma::Sides* s, pcu::PCU *PCUObj) { double tot = TO_DOUBLE(s->total()); - tot = PCUObj->Add(tot); + tot = PCUObj->Add(tot); return tot / PCUObj->Peers(); } diff --git a/parma/diffMC/parma_step.cc b/parma/diffMC/parma_step.cc index 298e041e1..9e1b5c49e 100644 --- a/parma/diffMC/parma_step.cc +++ b/parma/diffMC/parma_step.cc @@ -34,7 +34,7 @@ namespace parma { if ( stop->stop(imb,maxImb,m->getPCU()) ) return false; apf::Migration* plan = selects->run(targets); - int planSz = m->getPCU()->Add(plan->count()); + int planSz = m->getPCU()->Add(plan->count()); const double t0 = pcu::Time(); m->migrate(plan); if ( !m->getPCU()->Self() && verbosity ) diff --git a/parma/diffMC/parma_vtxPtnWriter.cc b/parma/diffMC/parma_vtxPtnWriter.cc index 3a77013a9..62763b551 100644 --- a/parma/diffMC/parma_vtxPtnWriter.cc +++ b/parma/diffMC/parma_vtxPtnWriter.cc @@ -21,7 +21,7 @@ namespace { } public: Ptn(apf::Mesh* m) { - const long totv = m->getPCU()->Add(countOwned(m)); + const long totv = m->getPCU()->Add(countOwned(m)); c = pp = totv / m->getPCU()->Peers(); f = pp * m->getPCU()->Self(); const int remainder = totv % m->getPCU()->Peers(); diff --git a/parma/parma.cc b/parma/parma.cc index 18dcba083..929c7db6c 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -92,9 +92,9 @@ namespace { double (*min)[4], double (*max)[4], double (*avg)[4]) { for(int d=0; d<=dim; d++) (*min)[d] = (*max)[d] = (*tot)[d] = (*loc)[d]; - PCUObj->Min(*min, dim+1); - PCUObj->Max(*max, dim+1); - PCUObj->Add(*tot, dim+1); + PCUObj->Min(*min, dim+1); + PCUObj->Max(*max, dim+1); + PCUObj->Add(*tot, dim+1); for(int d=0; d<=dim; d++) { (*avg)[d] = (*tot)[d]; (*avg)[d] /= TO_DOUBLE(PCUObj->Peers()); @@ -102,9 +102,9 @@ namespace { } void getStats(pcu::PCU *PCUObj, int& loc, long& tot, int& min, int& max, double& avg) { - min = PCUObj->Min(loc); - max = PCUObj->Max(loc); - tot = PCUObj->Add(TO_LONG(loc)); + min = PCUObj->Min(loc); + max = PCUObj->Max(loc); + tot = PCUObj->Add(TO_LONG(loc)); avg = TO_DOUBLE(tot) / PCUObj->Peers(); } @@ -167,8 +167,8 @@ void Parma_GetEntImbalance(apf::Mesh* mesh, double (*entImb)[4]) { dims = TO_SIZET(mesh->getDimension()) + 1; for(size_t i=0; i < dims; i++) tot[i] = (*entImb)[i] = mesh->count(TO_INT(i)); - mesh->getPCU()->Add(tot, dims); - mesh->getPCU()->Max(*entImb, dims); + mesh->getPCU()->Add(tot, dims); + mesh->getPCU()->Max(*entImb, dims); for(size_t i=0; i < dims; i++) (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) @@ -182,8 +182,8 @@ void Parma_GetWeightedEntImbalance(apf::Mesh* mesh, apf::MeshTag* w, double tot[4] = {0,0,0,0}; for(size_t i=0; i < dims; i++) tot[i] = (*entImb)[i]; - mesh->getPCU()->Add(tot, TO_SIZET(dims)); - mesh->getPCU()->Max(*entImb, TO_SIZET(dims)); + mesh->getPCU()->Add(tot, TO_SIZET(dims)); + mesh->getPCU()->Max(*entImb, TO_SIZET(dims)); for(size_t i=0; i < dims; i++) (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) @@ -199,8 +199,8 @@ double Parma_GetWeightedEntImbalance(apf::Mesh* m, apf::MeshTag* w, while ((e = m->iterate(it))) sum += getEntWeight(m, e, w); m->end(it); - double tot = m->getPCU()->Add(sum); - double max = m->getPCU()->Max(sum); + double tot = m->getPCU()->Add(sum); + double max = m->getPCU()->Max(sum); return max/(tot/m->getPCU()->Peers()); } @@ -209,9 +209,9 @@ void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, mii nborToShared; getNeighborCounts(m,nborToShared); loc = TO_INT(nborToShared.size())-1; - max = m->getPCU()->Max(loc); - avg = TO_DOUBLE(m->getPCU()->Add(loc)) / m->getPCU()->Peers(); - numMaxParts = m->getPCU()->Add( (int)(loc==max) ); + max = m->getPCU()->Max(loc); + avg = TO_DOUBLE(m->getPCU()->Add(loc)) / m->getPCU()->Peers(); + numMaxParts = m->getPCU()->Add( (int)(loc==max) ); } void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { @@ -223,7 +223,7 @@ void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { for(int i=0; isecond == i+1 ) smallCnt[i]++; - m->getPCU()->Add(smallCnt,small); + m->getPCU()->Add(smallCnt,small); if( !m->getPCU()->Self() ) { std::stringstream ss; for(int i=0; igetPCU()->Max(loc); + int max = m->getPCU()->Max(loc); int smallest = INT_MAX; if( loc == max ) { APF_ITERATE(mii, nborToShared, nbor) if( nbor->second < smallest ) smallest = nbor->second; } - return m->getPCU()->Min(smallest); + return m->getPCU()->Min(smallest); } void Parma_GetOwnedBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, @@ -270,8 +270,8 @@ void Parma_GetMdlBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, void Parma_GetDisconnectedStats(apf::Mesh* m, int& max, double& avg, int& loc) { dcPart dc(m); loc = TO_INT(dc.getNumDcComps()); - max = m->getPCU()->Max(loc); - avg = TO_DOUBLE( m->getPCU()->Add(loc) ) / m->getPCU()->Peers(); + max = m->getPCU()->Max(loc); + avg = TO_DOUBLE( m->getPCU()->Add(loc) ) / m->getPCU()->Peers(); } void Parma_ProcessDisconnectedParts(apf::Mesh* m) { @@ -328,13 +328,13 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, int surf = numSharedSides(m); double vol = TO_DOUBLE( m->count(m->getDimension()) ); double surfToVol = surf/vol; - double minSurfToVol = m->getPCU()->Min(surfToVol); - double maxSurfToVol = m->getPCU()->Max(surfToVol); - double avgSurfToVol = m->getPCU()->Add(surfToVol) / m->getPCU()->Peers(); + double minSurfToVol = m->getPCU()->Min(surfToVol); + double maxSurfToVol = m->getPCU()->Max(surfToVol); + double avgSurfToVol = m->getPCU()->Add(surfToVol) / m->getPCU()->Peers(); m->getPCU()->DebugPrint("%s sharedSidesToElements %.3f\n", key.c_str(), surfToVol); int empty = (m->count(m->getDimension()) == 0 ) ? 1 : 0; - empty = m->getPCU()->Add(empty); + empty = m->getPCU()->Add(empty); double imb[4] = {0, 0, 0, 0}; Parma_GetWeightedEntImbalance(m,w,&imb); @@ -419,7 +419,7 @@ int Parma_MisNumbering(apf::Mesh* m, int d) { } iter++; misSize = (misNumber != -1); - misSize = m->getPCU()->Add(misSize); + misSize = m->getPCU()->Add(misSize); } return misNumber; } diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index a40ada8c4..a720386c8 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -94,7 +94,7 @@ struct AdaptCallback : public Parma_GroupCode static double getAveragePartDensity(apf::Mesh* m) { double nElements = m->count(m->getDimension()); - nElements = m->getPCU()->Add(nElements); + nElements = m->getPCU()->Add(nElements); return nElements / m->getPCU()->Peers(); } diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 670473a53..6b168efca 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -33,9 +33,9 @@ static void print_stats(const char* name, double value, pcu::PCU *pcu_obj) { double min, max, avg; - min = pcu_obj->Min(value); - max = pcu_obj->Max(value); - avg = pcu_obj->Add(value); + min = pcu_obj->Min(value); + max = pcu_obj->Max(value); + avg = pcu_obj->Add(value); avg /= pcu_obj->Peers(); double imb = max / avg; if (!pcu_obj->Self()) diff --git a/phasta/phGrowthCurves.cc b/phasta/phGrowthCurves.cc index ef1e3248f..b61d74cd3 100644 --- a/phasta/phGrowthCurves.cc +++ b/phasta/phGrowthCurves.cc @@ -237,8 +237,8 @@ void getGrowthCurves(Output& o) lion_oprint(1,"%s: rank %d, ngc, nv: %d, %d\n", __func__, o.mesh->getPCU()->Self(), ngc, nv); - o.mesh->getPCU()->Add(&ngc,sizeof(ngc)); - o.mesh->getPCU()->Add(&nv,sizeof(nv)); + o.mesh->getPCU()->Add(&ngc,sizeof(ngc)); + o.mesh->getPCU()->Add(&nv,sizeof(nv)); if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"%s: total ngc, nv: %d, %d\n", __func__, ngc, nv); diff --git a/phasta/phInterfaceCutter.cc b/phasta/phInterfaceCutter.cc index 1b51cc809..76d5513ea 100644 --- a/phasta/phInterfaceCutter.cc +++ b/phasta/phInterfaceCutter.cc @@ -272,7 +272,7 @@ int migrateInterface(apf::Mesh2*& m, ph::BCs& bcs) { lion_oprint(1,"proc-%d: number of migrating elements: %d\n",m->getPCU()->Self(),plan->count()); int totalPlan = plan->count(); - totalPlan = m->getPCU()->Add(totalPlan); + totalPlan = m->getPCU()->Add(totalPlan); m->migrate(plan); return totalPlan; diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc index 35b09fb96..65bd3905f 100644 --- a/phasta/phOutput.cc +++ b/phasta/phOutput.cc @@ -31,11 +31,11 @@ static void getCounts(Output& o) static void checkLoadBalance(Output& o) { - long sumOwnedNodes = o.mesh->getPCU()->Add(o.nOwnedNodes); - long sumAllNodes = o.mesh->getPCU()->Add(o.nOverlapNodes); + long sumOwnedNodes = o.mesh->getPCU()->Add(o.nOwnedNodes); + long sumAllNodes = o.mesh->getPCU()->Add(o.nOverlapNodes); double avgNodes = static_cast(sumAllNodes) / o.mesh->getPCU()->Peers(); double vlbratio = o.nOverlapNodes / avgNodes; - double vlbratio_max = o.mesh->getPCU()->Max(vlbratio); + double vlbratio_max = o.mesh->getPCU()->Max(vlbratio); if (!o.mesh->getPCU()->Self()) lion_oprint(1,"max vertex load imbalance of partitioned mesh = %f\n", vlbratio_max); if (!o.mesh->getPCU()->Self()) @@ -43,10 +43,10 @@ static void checkLoadBalance(Output& o) int dim = o.mesh->getDimension(); int numElms = o.mesh->count(dim); - long sumElms = o.mesh->getPCU()->Add(numElms); + long sumElms = o.mesh->getPCU()->Add(numElms); double avgElms = static_cast(sumElms) / o.mesh->getPCU()->Peers(); double elbratio = numElms / avgElms; - double elbratio_max = o.mesh->getPCU()->Max(elbratio); + double elbratio_max = o.mesh->getPCU()->Max(elbratio); if (!o.mesh->getPCU()->Self()) lion_oprint(1,"max region (3D) or face (2D) load imbalance of partitioned mesh = %f\n", elbratio_max); } @@ -369,7 +369,7 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { rbIDmap[rrbID] = rrbMT; } - int rbIDs_size = m->getPCU()->Max(rbIDmap.size()); + int rbIDs_size = m->getPCU()->Max(rbIDmap.size()); int* rbIDs = new int[rbIDs_size](); int* rbMTs = new int[rbIDs_size](); @@ -384,8 +384,8 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { } // allreduce the set - m->getPCU()->Max(rbIDs, rbIDs_size); - m->getPCU()->Max(rbMTs, rbIDs_size); + m->getPCU()->Max(rbIDs, rbIDs_size); + m->getPCU()->Max(rbMTs, rbIDs_size); // attach data o.numRigidBody = rbIDs_size; diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc index 25a52c436..0d310e8ca 100644 --- a/phasta/phPartition.cc +++ b/phasta/phPartition.cc @@ -93,7 +93,7 @@ bool isMixed(apf::Mesh2* m) { break; } m->end(it); - return m->getPCU()->Max(mixed); + return m->getPCU()->Max(mixed); } void clearTags(apf::Mesh* m, apf::MeshTag* t) { diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index e966fc8c8..2c5bd6590 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -189,11 +189,11 @@ static void fixCoords(apf::Mesh2* m) double global_max[2]; global_max[0] = max_x_diff; global_max[1] = max_p_diff; - m->getPCU()->Max(global_max, 2); + m->getPCU()->Max(global_max, 2); long global_diffs[2]; global_diffs[0] = x_diffs; global_diffs[1] = p_diffs; - m->getPCU()->Add(global_diffs, 2); + m->getPCU()->Add(global_diffs, 2); /* admittedly not the best way of checking which processor had the max */ if (global_diffs[0] && (global_max[0] == max_x_diff)) diff --git a/pumi/pumi_field.cc b/pumi/pumi_field.cc index 9f4707c8e..77d0e0398 100644 --- a/pumi/pumi_field.cc +++ b/pumi/pumi_field.cc @@ -494,7 +494,7 @@ void pumi_field_verify(pMesh m, pField f, pOwnership shr) receiveFieldData(fields,mismatch_fields,m->getPCU()); } } - int global_size = m->getPCU()->Max((int)mismatch_fields.size()); + int global_size = m->getPCU()->Max((int)mismatch_fields.size()); if (global_size) { if (!m->getPCU()->Self()) diff --git a/sam/samElementCount.cc b/sam/samElementCount.cc index 10a6bf71e..a5ac6086a 100644 --- a/sam/samElementCount.cc +++ b/sam/samElementCount.cc @@ -52,7 +52,7 @@ class TotalMetricVolumeIso : public apf::Integrator { sum += vhat * w * dV; } virtual void parallelReduce(pcu::PCU *PCUObj) { - sum = PCUObj->Add(sum); + sum = PCUObj->Add(sum); } }; diff --git a/spr/sprEstimateError.cc b/spr/sprEstimateError.cc index c80a16cb0..523ef4134 100644 --- a/spr/sprEstimateError.cc +++ b/spr/sprEstimateError.cc @@ -26,7 +26,7 @@ class SInt : public apf::Integrator {} void parallelReduce(pcu::PCU *PCUObj) { - PCUObj->Add(&r,1); + PCUObj->Add(&r,1); } void reset() {r=0;} double r; diff --git a/spr/sprEstimateTargetError.cc b/spr/sprEstimateTargetError.cc index 698a600a3..84b3183d1 100644 --- a/spr/sprEstimateTargetError.cc +++ b/spr/sprEstimateTargetError.cc @@ -66,7 +66,7 @@ class ScalarIntegrator : public apf::Integrator } void parallelReduce(pcu::PCU *PCUObj) { - PCUObj->Add(&result,1); + PCUObj->Add(&result,1); } double result; }; diff --git a/test/curvetest.cc b/test/curvetest.cc index 90369224e..7f7c52223 100644 --- a/test/curvetest.cc +++ b/test/curvetest.cc @@ -77,7 +77,7 @@ static void testElementSize(apf::Mesh* m) m->end(it); } - m->getPCU()->Add(&(sizes[0]),3); + m->getPCU()->Add(&(sizes[0]),3); if(!m->getPCU()->Self()) printf("Total sizes for order %d %f %f %f\n", m->getShape()->getOrder(), diff --git a/test/describe.cc b/test/describe.cc index 0fc79146f..14e6c376d 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -23,9 +23,9 @@ static void print_stats(const char* name, double value, pcu::PCU *PCUObj) { double min, max, avg; - min = PCUObj->Min(value); - max = PCUObj->Max(value); - avg = PCUObj->Add(value); + min = PCUObj->Min(value); + max = PCUObj->Max(value); + avg = PCUObj->Add(value); avg /= PCUObj->Peers(); double imb = max / avg; if (!PCUObj->Self()) diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index 06ed71abb..d1508db23 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -682,7 +682,7 @@ void readMesh(const char* meshfilename, PCU_ALWAYS_ASSERT(fc); getNumVerts(fc,mesh.numVerts); mesh.localNumVerts=mesh.numVerts; - mesh.numVerts=PCUObj->Add(mesh.numVerts); + mesh.numVerts=PCUObj->Add(mesh.numVerts); if(!PCUObj->Self()) fprintf(stderr, "numVerts %ld\n", mesh.numVerts); diff --git a/test/quality.cc b/test/quality.cc index f5c472581..90fc9586b 100644 --- a/test/quality.cc +++ b/test/quality.cc @@ -39,14 +39,14 @@ static void getConfig(int argc, char** argv, pcu::PCU *PCUObj) static void parallelReduce(apf::Mesh2* m) { long numElems = m->count(m->getDimension()); - numElems = m->getPCU()->Add(numElems); - avgQuality = m->getPCU()->Add(avgQuality); + numElems = m->getPCU()->Add(numElems); + avgQuality = m->getPCU()->Add(avgQuality); avgQuality /= numElems; - numElemsBelowTol = m->getPCU()->Add(numElemsBelowTol); - avgQualityBelowTol = m->getPCU()->Add(avgQualityBelowTol); + numElemsBelowTol = m->getPCU()->Add(numElemsBelowTol); + avgQualityBelowTol = m->getPCU()->Add(avgQualityBelowTol); if (numElemsBelowTol > 0) avgQualityBelowTol /= numElemsBelowTol; - minQuality = m->getPCU()->Min(minQuality); + minQuality = m->getPCU()->Min(minQuality); } static void processMesh(apf::Mesh2* m) diff --git a/test/visualizeAnisoSizes.cc b/test/visualizeAnisoSizes.cc index 63511064b..972b2eb60 100644 --- a/test/visualizeAnisoSizes.cc +++ b/test/visualizeAnisoSizes.cc @@ -122,7 +122,7 @@ double getLargetsSize( maxSize = scales[2]; } m->end(it); - m->getPCU()->Max(&maxSize, 1); + m->getPCU()->Max(&maxSize, 1); return maxSize; } From 6faa7654daa25dbaf7d83311dd9e258ed82690e4 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 19 Aug 2024 17:46:05 -0400 Subject: [PATCH 134/141] Capstone updated to use PCU Object --- apf_cap/apfCAP.cc | 4 +- apf_cap/apfCAP.h | 3 +- capstone_clis/capStone2VTK.cc | 12 ++--- capstone_clis/capStoneAnalyzeArea.cc | 14 +++--- capstone_clis/capStoneAnalyzeEdgeLengths.cc | 14 +++--- capstone_clis/capStoneAnisoAdapt.cc | 12 ++--- capstone_clis/capStoneAnisoAdaptWing.cc | 12 ++--- capstone_clis/capStoneAttachSolution.cc | 42 +++++++++--------- capstone_clis/capStoneAttachSolution2.cc | 44 ++++++++++--------- .../capStoneAttachSolutionStrandOnly.cc | 42 +++++++++--------- capstone_clis/capStoneAttachSolutionUni.cc | 30 +++++++------ capstone_clis/capStoneCheckParametrization.cc | 10 +++-- capstone_clis/capStoneCurve.cc | 10 +++-- capstone_clis/capStoneGeomTest.cc | 32 +++++++------- capstone_clis/capStoneIsoAdaptB737.cc | 12 ++--- test/capVol.cc | 13 +++--- 16 files changed, 167 insertions(+), 139 deletions(-) diff --git a/apf_cap/apfCAP.cc b/apf_cap/apfCAP.cc index 2cc0b9cc0..31985c432 100644 --- a/apf_cap/apfCAP.cc +++ b/apf_cap/apfCAP.cc @@ -934,10 +934,10 @@ MeshEntity* castEntity(capEntity* entity) return 0; } -Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb) +Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb, pcu::PCU *PCUObj) { MeshCAP* m = new MeshCAP(mdb, gdb); - m->init(getLagrange(1)); + m->init(getLagrange(1), PCUObj); return m; } diff --git a/apf_cap/apfCAP.h b/apf_cap/apfCAP.h index 102716bed..0c6279012 100644 --- a/apf_cap/apfCAP.h +++ b/apf_cap/apfCAP.h @@ -2,6 +2,7 @@ #define APFCAP #include +#include #include "CapstoneModule.h" #include "CreateMG_Framework_Geometry.h" @@ -24,7 +25,7 @@ namespace apf { * * \details This object should be destroyed by apf::destroyMesh. */ -Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb); +Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb, pcu::PCU *PCUObj); /** * \brief Casts a CapStone entity to an apf::MeshEntity. diff --git a/capstone_clis/capStone2VTK.cc b/capstone_clis/capStone2VTK.cc index 8c4124cdc..a552797a3 100644 --- a/capstone_clis/capStone2VTK.cc +++ b/capstone_clis/capStone2VTK.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -21,6 +21,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -41,10 +42,11 @@ using namespace CreateMG::Geometry; int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -131,11 +133,11 @@ int main(int argc, char** argv) gmi_register_cap(); // convert the mesh to apf/mds mesh - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); apf::writeVtkFiles(folderName, mesh); gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnalyzeArea.cc b/capstone_clis/capStoneAnalyzeArea.cc index 86f83634b..c535c0616 100644 --- a/capstone_clis/capStoneAnalyzeArea.cc +++ b/capstone_clis/capStoneAnalyzeArea.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -48,12 +49,13 @@ double addSizeDistribution(apf::Mesh2* m, int factor, int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -143,7 +145,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); printf("---- Creating Mesh Wrapper Object: Done. \n"); // add size distribution based on area @@ -174,7 +176,7 @@ int main(int argc, char** argv) printf("total bin count is %lu \n", totalBinCount); gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnalyzeEdgeLengths.cc b/capstone_clis/capStoneAnalyzeEdgeLengths.cc index ffce0c255..7e4881038 100644 --- a/capstone_clis/capStoneAnalyzeEdgeLengths.cc +++ b/capstone_clis/capStoneAnalyzeEdgeLengths.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -49,12 +50,13 @@ void edgeLengthStats(apf::Mesh2* m); int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -144,7 +146,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); printf("---- Creating Mesh Wrapper Object: Done. \n"); // add size distribution based on area @@ -153,7 +155,7 @@ int main(int argc, char** argv) edgeLengthStats(mesh); gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnisoAdapt.cc b/capstone_clis/capStoneAnisoAdapt.cc index 1e0c4949c..2e7329c52 100644 --- a/capstone_clis/capStoneAnisoAdapt.cc +++ b/capstone_clis/capStoneAnisoAdapt.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -11,6 +11,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -49,10 +50,11 @@ int main(int argc, char** argv) { /* INIT CALLS */ MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc < 3) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE: %s \n", argv[0]); printf("Size-fields:\n"); printf("%d, for uniform anisotropic size-field\n", 1); @@ -110,7 +112,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); @@ -197,6 +199,6 @@ int main(int argc, char** argv) /* EXIT CALLS */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnisoAdaptWing.cc b/capstone_clis/capStoneAnisoAdaptWing.cc index 39e7f53d4..92d481459 100644 --- a/capstone_clis/capStoneAnisoAdaptWing.cc +++ b/capstone_clis/capStoneAnisoAdaptWing.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -11,6 +11,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -93,10 +94,11 @@ int main(int argc, char** argv) { /* INIT CALLS */ MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc < 2) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -149,7 +151,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); @@ -223,6 +225,6 @@ int main(int argc, char** argv) /* EXIT CALLS */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAttachSolution.cc b/capstone_clis/capStoneAttachSolution.cc index 2839c0772..4203d66c9 100644 --- a/capstone_clis/capStoneAttachSolution.cc +++ b/capstone_clis/capStoneAttachSolution.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -149,8 +150,8 @@ int gradeSizeModify(apf::Mesh* m, apf::Field* size_iso,double gradingFactor, m->getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -252,7 +253,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); + m->getPCU()->Add(&needsParallel,1); m->getPCU()->Send(); apf::MeshEntity* ent; @@ -267,8 +268,8 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) //owning copies are receiving while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } @@ -315,7 +316,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -598,12 +599,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList, int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 9) { - if(0==PCU_Comm_Self()) { + if(0==PCUObj.get()->Self()) { std::cerr << "usage: " << argv[0] << " \n"; std::cerr << "*target field(s) is a bit string to select which field(s) are used for error estimation\n"; @@ -711,7 +713,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); printf("---- Creating Mesh Wrapper Object: Done. \n"); // make the volume mesh (this one is MDS underneath) @@ -735,7 +737,7 @@ int main(int argc, char** argv) printf("number of mesh regions(hexes): %zu\n", volMesh->count(3)); printf("---- Printing Volume/Strand Mesh Stats: Done. \n"); - double constructionTime = PCU_Time(); + double constructionTime = pcu::Time(); std::cout<<"TIMER: Finished converting capstone mesh to volume mesh "<end(itVol); std::cout<<"Finished surface fields from volume\n"; - double getSurfaceTime = PCU_Time(); + double getSurfaceTime = pcu::Time(); std::cout<<"TIMER: Finished computing speed and transferring fields to surface "<debugFolder = "debug"; */ ma::adaptVerbose(in, false); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); std::cout<<"TIMER: adaptMesh "< +#include #include #include #include @@ -20,6 +20,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -121,8 +122,8 @@ int gradeSizeModify(apf::Mesh* m, apf::Field* size_iso,double gradingFactor, m->getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -224,7 +225,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); + m->getPCU()->Add(&needsParallel,1); m->getPCU()->Send(); apf::MeshEntity* ent; @@ -239,8 +240,8 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) //owning copies are receiving while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } @@ -287,7 +288,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -493,12 +494,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList,apf: int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -597,7 +599,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); printf("---- Creating Mesh Wrapper Object: Done. \n"); // remove unused verts @@ -628,7 +630,7 @@ int main(int argc, char** argv) printf("number of mesh regions(hexes): %zu\n", volMesh->count(3)); printf("---- Printing Volume/Strand Mesh Stats: Done. \n"); - double constructionTime = PCU_Time(); + double constructionTime = pcu::Time(); std::cout<<"TIMER: Finished converting capstone mesh to volume mesh "<end(itVol); std::cout<<"Finished surface fields from volume\n"; - double getSurfaceTime = PCU_Time(); + double getSurfaceTime = pcu::Time(); std::cout<<"TIMER: Finished computing speed and transferring fields to surface "<shouldForceAdaptation = true; ma::adaptVerbose(in); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); std::cout<<"TIMER: adaptMesh "< +#include #include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -147,8 +148,8 @@ int gradeSizeModify(apf::Mesh* m, apf::Field* size_iso,double gradingFactor, m->getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -250,7 +251,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); + m->getPCU()->Add(&needsParallel,1); m->getPCU()->Send(); apf::MeshEntity* ent; @@ -265,8 +266,8 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) //owning copies are receiving while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } @@ -313,7 +314,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -520,12 +521,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList, con int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCU_Comm_Self()) { + if(0==PCUObj.get()->Self()) { std::cerr << "usage: " << argv[0] << " \n"; std::cerr << "*target field(s) is a bit string to select which field(s) are used for error estimation\n"; @@ -556,7 +558,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); printf("---- Creating Mesh Wrapper Object: Done. \n"); // make the volume mesh (this one is MDS underneath) @@ -581,7 +583,7 @@ int main(int argc, char** argv) printf("number of mesh regions(hexes): %zu\n", volMesh->count(3)); printf("---- Printing Volume/Strand Mesh Stats: Done. \n"); - double constructionTime = PCU_Time(); + double constructionTime = pcu::Time(); std::cout<<"TIMER: Finished converting capstone mesh to volume mesh "<end(itVol); std::cout<<"Finished surface fields from volume\n"; - double getSurfaceTime = PCU_Time(); + double getSurfaceTime = pcu::Time(); std::cout<<"TIMER: Finished computing speed and transferring fields to surface "<debugFolder = "debug"; */ ma::adaptVerbose(in, false); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); std::cout<<"TIMER: adaptMesh "< +#include #include #include #include @@ -22,6 +22,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -125,8 +126,8 @@ int gradeSizeModify(apf::Mesh* m, apf::Field* size_iso,double gradingFactor, m->getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -228,7 +229,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); + m->getPCU()->Add(&needsParallel,1); m->getPCU()->Send(); apf::MeshEntity* ent; @@ -243,8 +244,8 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) //owning copies are receiving while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } @@ -291,7 +292,7 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -497,12 +498,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList,apf: int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -594,7 +596,7 @@ int main(int argc, char** argv) printf("---- CapStone Mesh Loaded. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); //adapt the mesh ma::Input* in; @@ -612,7 +614,7 @@ int main(int argc, char** argv) in->debugFolder = "debug"; ma::adaptVerbose(in, true); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); // write the adapted mesh to vtk @@ -635,7 +637,7 @@ int main(int argc, char** argv) } gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneCheckParametrization.cc b/capstone_clis/capStoneCheckParametrization.cc index 64bf03aba..5273f708b 100644 --- a/capstone_clis/capStoneCheckParametrization.cc +++ b/capstone_clis/capStoneCheckParametrization.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -7,6 +7,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -28,10 +29,11 @@ void checkParametrization(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc != 2) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.get()->Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -112,7 +114,7 @@ int main(int argc, char** argv) // check parametrization using capstone apis checkParametrization(m, g); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneCurve.cc b/capstone_clis/capStoneCurve.cc index 8f2da9ff8..17e62261b 100644 --- a/capstone_clis/capStoneCurve.cc +++ b/capstone_clis/capStoneCurve.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -13,6 +13,7 @@ #include #include #include +#include // === includes for safe_mkdir === #include @@ -101,7 +102,8 @@ void writeVtk(CapstoneModule& cs, const std::string& vtkFileName) int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); @@ -160,7 +162,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* writeVtk(cs, "before.vtk"); */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); apf::MeshEntity* ent; @@ -195,6 +197,6 @@ int main(int argc, char** argv) /* writeVtk(cs, "after.vtk"); */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneGeomTest.cc b/capstone_clis/capStoneGeomTest.cc index fcc921c20..13cc05dd9 100644 --- a/capstone_clis/capStoneGeomTest.cc +++ b/capstone_clis/capStoneGeomTest.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -15,6 +15,7 @@ #include #include #include +#include #include "CapstoneModule.h" @@ -39,15 +40,16 @@ char const* const typeName[4] = "region"}; void printInfo(gmi_model* model, int dim); -void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName); -void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName); -void visualizeEdges(gmi_model* model, int n, const char* fileName); +void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName, pcu::PCU *PCUObj); +void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName, pcu::PCU *PCUObj); +void visualizeEdges(gmi_model* model, int n, const char* fileName, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); gmi_register_mesh(); gmi_register_null(); @@ -115,7 +117,7 @@ int main(int argc, char** argv) gmi_register_cap(); - apf::Mesh2* mesh0 = apf::createMesh(m,g); + apf::Mesh2* mesh0 = apf::createMesh(m,g,PCUObj.get()); apf::writeVtkFiles("mesh_no_param", mesh0); gmi_model* model = gmi_import_cap(g); @@ -136,7 +138,7 @@ int main(int argc, char** argv) while( (ge = gmi_next(model, gi)) ){ std::stringstream name_str; name_str << "face_" << gmi_tag(model, ge) << "_mesh"; - visualizeFace(model, ge, 100, 100, name_str.str().c_str()); + visualizeFace(model, ge, 100, 100, name_str.str().c_str(), PCUObj.get()); } gmi_end(model, gi); @@ -144,7 +146,7 @@ int main(int argc, char** argv) printf("creating mesh with param field\n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); apf::Field* pf = apf::createFieldOn(mesh, "param_field", apf::VECTOR); apf::Field* idf = apf::createFieldOn(mesh, "id", apf::SCALAR); apf::MeshEntity* e; @@ -162,7 +164,7 @@ int main(int argc, char** argv) gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -236,7 +238,7 @@ void printInfo(gmi_model* model, int dim) printf("-------------------------------------------\n"); } -void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName) +void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName, pcu::PCU *PCUObj) { // assert type is 2 // get the range @@ -270,7 +272,7 @@ void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* // make the vertexes and set the coordinates using the array std::vector vs; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, PCUObj.get()); for (size_t i = 0; i < ps.size(); i++) { ma::Entity* vert = mesh->createVert(0); mesh->setPoint(vert, 0, ps[i]); @@ -341,7 +343,7 @@ void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* } -void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName) +void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName, pcu::PCU *PCUObj) { // assert type is 1 // get the range @@ -365,7 +367,7 @@ void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileNam // make the vertexes and set the coordinates using the array std::vector vs; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, pcu::PCU *PCUObj); for (size_t i = 0; i < ps.size(); i++) { ma::Entity* vert = mesh->createVert(0); mesh->setPoint(vert, 0, ps[i]); @@ -404,9 +406,9 @@ void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileNam apf::destroyMesh(mesh); } -void visualizeEdges(gmi_model* model, int n, const char* fileName) +void visualizeEdges(gmi_model* model, int n, const char* fileName, pcu::PCU *PCUObj) { - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, PCUObj); gmi_ent* entity; gmi_iter* gi = gmi_begin(model, 1); while( (entity = gmi_next(model, gi)) ){ diff --git a/capstone_clis/capStoneIsoAdaptB737.cc b/capstone_clis/capStoneIsoAdaptB737.cc index 819ce55d8..7ee4aa361 100644 --- a/capstone_clis/capStoneIsoAdaptB737.cc +++ b/capstone_clis/capStoneIsoAdaptB737.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -14,6 +14,7 @@ #include #include #include +#include #include "CapstoneModule.h" /* #include "CreateMG_Framework_Core.h" */ @@ -217,10 +218,11 @@ int main(int argc, char** argv) { /* INIT CALLS */ MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc < 2) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -273,7 +275,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); @@ -374,6 +376,6 @@ int main(int argc, char** argv) /* EXIT CALLS */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/capVol.cc b/test/capVol.cc index 24c99483d..25b38091e 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -1,11 +1,12 @@ #include #include +#include // Output #include // Parallelism -#include +#include #include // Mesh interfaces @@ -30,7 +31,6 @@ namespace { void myExit(int exit_code = EXIT_SUCCESS) { gmi_cap_stop(); - PCU_Comm_Free(); MPI_Finalize(); exit(exit_code); } @@ -77,7 +77,8 @@ void printUsage(char *argv0) { int main(int argc, char** argv) { // Initialize parallelism. MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); // Initialize logging. lion_set_stdout(stdout); @@ -85,7 +86,7 @@ int main(int argc, char** argv) { // Check arguments or print usage. if (argc < 3) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.get()->Self() == 0) { printUsage(argv[0]); } myExit(EXIT_FAILURE); @@ -173,7 +174,7 @@ int main(int argc, char** argv) { MG_API_CALL(m, compute_adjacency()); // Make APF adapter over Capstone mesh. - ma::Mesh* apfCapMesh = apf::createMesh(m, g); + ma::Mesh* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); // Choose appropriate size-field. ma::AnisotropicFunction* sf = nullptr; @@ -271,6 +272,6 @@ int main(int argc, char** argv) { // Exit calls. gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } \ No newline at end of file From b5a2190cf900e029b1f1824bd0801254d7c1475b Mon Sep 17 00:00:00 2001 From: flagdanger Date: Wed, 21 Aug 2024 12:47:20 -0400 Subject: [PATCH 135/141] small fixes --- apf/apfCGNS.cc | 2 +- apf/apfCavityOp.cc | 2 +- apf/apfConstruct.cc | 4 ++-- apf/apfConvertTags.h | 2 +- apf/apfMixedNumbering.cc | 2 +- apf/apfNumbering.cc | 2 +- crv/crvShape.cc | 8 ++++---- parma/parma.cc | 4 ++-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 71695d719..92eb98782 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -633,7 +633,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, int startOfBCBlock = startingLocation + 1; const int number = bc.second.size(); int total = number; - m->getPCU()->AddgetPCU()->Add(&total, 1); // size of total array if (total > 0) { const auto allEnd = startOfBCBlock + total - 1; //one-based diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 146f0a084..568fd2287 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -133,7 +133,7 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = mesh->getPCU()->Min(static_cast(requests.empty())); + int done = mesh->getPCU()->Min(requests.empty()); if (done) return false; /* throw in the local pull requests */ int self = mesh->getPCU()->Self(); diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index 8c2a25c61..39afa765d 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -227,7 +227,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, This means we might need to send and recv some coords */ double* c = new double[mySize*3]; - Gid start = m->getPCU()->Exscan(nverts); + Gid start = m->getPCU()->Exscan(nverts); m->getPCU()->Begin(); // the forced 64 bit math below may not be necessary Gid tmpL=start / quotient; @@ -317,7 +317,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, /* Force each peer to have exactly mySize verts. This means we might need to send and recv some matches */ Gid* c = new Gid[mySize]; - Gid start = m->getPCU()->Exscan(nverts); + Gid start = m->getPCU()->Exscan(nverts); m->getPCU()->Begin(); diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h index e4dbd446b..0cbf63ba2 100644 --- a/apf/apfConvertTags.h +++ b/apf/apfConvertTags.h @@ -86,7 +86,7 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, This means we might need to send and recv some coords */ T* c = new T[mySize*entries]; - apf::Gid start = m->getPCU()->Exscan(nverts); + apf::Gid start = m->getPCU()->Exscan(nverts); m->getPCU()->Begin(); apf::Gid tmpL=start / quotient; diff --git a/apf/apfMixedNumbering.cc b/apf/apfMixedNumbering.cc index 363c1e509..ae50bcf79 100644 --- a/apf/apfMixedNumbering.cc +++ b/apf/apfMixedNumbering.cc @@ -229,7 +229,7 @@ static void globalize( std::vector const& owned, std::vector& global, pcu::PCU *PCUObj) { - long start = PCUObj->Exscan(long(dofs)); + long start = PCUObj->Exscan(dofs); DynamicArray nodes; for (size_t f=0; f < global.size(); ++f) { getNodes(owned[f], nodes); diff --git a/apf/apfNumbering.cc b/apf/apfNumbering.cc index 2d7d408cc..345fb8701 100644 --- a/apf/apfNumbering.cc +++ b/apf/apfNumbering.cc @@ -537,7 +537,7 @@ int getElementNumbers(GlobalNumbering* n, MeshEntity* e, static long exscan(long x, pcu::PCU *PCUObj) { - return PCUObj->Exscan(x); + return PCUObj->Exscan(x); } template diff --git a/crv/crvShape.cc b/crv/crvShape.cc index 2001671cb..6bfc91768 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -634,7 +634,7 @@ static int markEdgesOppLargeAnglesTri(Adapt* a) } m->end(it); } while(count > prev_count); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } static int markEdgesOppLargeAnglesTet(Adapt* a) @@ -662,7 +662,7 @@ static int markEdgesOppLargeAnglesTet(Adapt* a) } m->end(it); } while(count > prev_count); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } /* The whole idea is to do the quality check once, @@ -700,7 +700,7 @@ static int markEdgesToFix(Adapt* a, int flag) } m->end(it); - return m->getPCU()->Add(count); + return m->getPCU()->Add(count); } int fixLargeBoundaryAngles(Adapt* a) @@ -732,7 +732,7 @@ static void collapseInvalidEdges(Adapt* a) findIndependentSet(a); successCount += ma::collapseAllEdges(a, modelDimension); } - successCount = m->getPCU()->Add(successCount); + successCount = m->getPCU()->Add(successCount); double t1 = pcu::Time(); ma::print(m->getPCU(), "Collapsed %d bad edges " "in %f seconds", successCount, t1-t0); diff --git a/parma/parma.cc b/parma/parma.cc index 929c7db6c..5be650b63 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -210,8 +210,8 @@ void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, getNeighborCounts(m,nborToShared); loc = TO_INT(nborToShared.size())-1; max = m->getPCU()->Max(loc); - avg = TO_DOUBLE(m->getPCU()->Add(loc)) / m->getPCU()->Peers(); - numMaxParts = m->getPCU()->Add( (int)(loc==max) ); + avg = TO_DOUBLE(m->getPCU()->Add(loc)) / m->getPCU()->Peers(); + numMaxParts = m->getPCU()->Add( (loc==max) ); } void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { From 85be809e9daffc1a8e47d5183d74d184d1d2250f Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 26 Aug 2024 13:13:47 -0400 Subject: [PATCH 136/141] Typo cleaning --- apf/apfConstruct.cc | 10 +++++----- mds/apfMDS.cc | 10 +++++----- parma/parma.cc | 2 +- pumi/pumi_mesh.cc | 2 +- test/fusion2.cc | 3 --- test/matchedNodeElmReader.cc | 2 +- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index 39afa765d..c72830f25 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -42,12 +42,12 @@ static NewElements constructElements( return newElements; } -static Gid getMax(const GlobalToVert& globalToVert, Mesh2* m) +static Gid getMax(const GlobalToVert& globalToVert, pcu::PCU *PCUObj) { Gid max = -1; APF_CONST_ITERATE(GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return m->getPCU()->Max(max); // this is type-dependent + return PCUObj->Max(max); // this is type-dependent } @@ -66,7 +66,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) } ifirst++; } - Gid max = getMax(globalToVert, m); // seems like we read this and know it already on every rank so why compute with global comm? + Gid max = getMax(globalToVert, m->getPCU()); // seems like we read this and know it already on every rank so why compute with global comm? ifirst=0; APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; @@ -212,7 +212,7 @@ NewElements construct(Mesh2* m, const Gid* conn, int nelem, int etype, void setCoords(Mesh2* m, const double* coords, int nverts, GlobalToVert& globalToVert) { - Gid max = getMax(globalToVert, m); + Gid max = getMax(globalToVert, m->getPCU()); Gid total = max + 1; int peers = m->getPCU()->Peers(); int quotient = total / peers; @@ -303,7 +303,7 @@ void setCoords(Mesh2* m, const double* coords, int nverts, void setMatches(Mesh2* m, const Gid* matches, int nverts, GlobalToVert& globalToVert) { - Gid max = getMax(globalToVert, m); + Gid max = getMax(globalToVert, m->getPCU()); Gid total = max + 1; int peers = m->getPCU()->Peers(); int quotient = total / peers; diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 65039867c..3b1c9bea9 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -198,8 +198,8 @@ class MeshMDS : public Mesh2 MeshMDS(gmi_model* m, const char* pathname, pcu::PCU *PCUObj) { init(apf::getLagrange(1), PCUObj); - mesh = mds_read_smb(this->getPCU()->GetCHandle(), m, pathname, 0, this); - isMatched = this->getPCU()->Or(!mds_net_empty(&mesh->matches)); + mesh = mds_read_smb(getPCU()->GetCHandle(), m, pathname, 0, this); + isMatched = getPCU()->Or(!mds_net_empty(&mesh->matches)); ownsModel = true; } ~MeshMDS() @@ -579,14 +579,14 @@ class MeshMDS : public Mesh2 } int getId() { - return this->getPCU()->Self(); + return getPCU()->Self(); } void writeNative(const char* fileName) { double t0 = pcu::Time(); - mesh = mds_write_smb(this->getPCU()->GetCHandle(), mesh, fileName, 0, this); + mesh = mds_write_smb(getPCU()->GetCHandle(), mesh, fileName, 0, this); double t1 = pcu::Time(); - if (!this->getPCU()->Self()) + if (!getPCU()->Self()) lion_oprint(1,"mesh %s written in %f seconds\n", fileName, t1 - t0); } void destroyNative() diff --git a/parma/parma.cc b/parma/parma.cc index 5be650b63..636a1ffc5 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -271,7 +271,7 @@ void Parma_GetDisconnectedStats(apf::Mesh* m, int& max, double& avg, int& loc) { dcPart dc(m); loc = TO_INT(dc.getNumDcComps()); max = m->getPCU()->Max(loc); - avg = TO_DOUBLE( m->getPCU()->Add(loc) ) / m->getPCU()->Peers(); + avg = TO_DOUBLE( m->getPCU()->Add(loc) ) / m->getPCU()->Peers(); } void Parma_ProcessDisconnectedParts(apf::Mesh* m) { diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 09b4c3745..587e87a65 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -69,7 +69,7 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) ++num_own; m->end(it); - m->getPCU()->Exscan(&num_own,1); + m->getPCU()->Exscan(&num_own,1); int initial_id=num_own; m->getPCU()->Begin(); diff --git a/test/fusion2.cc b/test/fusion2.cc index 511866a34..33fb8a407 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -92,7 +92,6 @@ struct GroupCode : public Parma_GroupCode apf::Mesh2* mesh; void run(int group) { - //mesh = ::makeEmptyMesh(PCUObj); if (group == 0) { addOneTri(mesh); setValues(mesh); @@ -150,8 +149,6 @@ int main( int argc, char* argv[]) int const groupSize = 1; apf::Unmodulo outMap(PCUObj.get()->Self(), groupSize); Parma_SplitPartition(code.mesh, groupSize, code, PCUObj.get()); - /* update mesh for leaving groups */ - //apf::remapPartition(code.mesh, outMap); globalCode(code.mesh); } MPI_Finalize(); diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index d1508db23..f0f8b3a4e 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -438,7 +438,7 @@ void getLocalRange(apf::Gid total, int& local, local += lpd; } } - first = PCUObj->Exscan(local); + first = PCUObj->Exscan(local); last = first+local; } From d5b74137b0972d1e6aaa94b75ce2a2b303e12e04 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 26 Aug 2024 15:11:26 -0400 Subject: [PATCH 137/141] Test files changed to not use unique_ptr --- test/1d.cc | 5 ++--- test/H1Shapes.cc | 11 +++++------ test/L2Shapes.cc | 7 +++---- test/align.cc | 9 ++++----- test/aniso_ma_test.cc | 5 ++--- test/ansys.cc | 7 +++---- test/assert_timing.cc | 3 +-- test/balance.cc | 5 ++--- test/bezierElevation.cc | 9 ++++----- test/bezierMesh.cc | 9 ++++----- test/bezierMisc.cc | 3 +-- test/bezierRefine.cc | 7 +++---- test/bezierShapeEval.cc | 7 +++---- test/bezierSubdivision.cc | 11 +++++------ test/bezierValidity.cc | 7 +++---- test/box.cc | 11 +++++------ test/capVol.cc | 7 +++---- test/cgns.cc | 15 +++++++-------- test/classifyThenAdapt.cc | 5 ++--- test/collapse.cc | 9 ++++----- test/construct.cc | 7 +++---- test/constructThenGhost.cc | 9 ++++----- test/construct_bottom_up.cc | 7 +++---- test/convert.cc | 13 ++++++------- test/crack_test.cc | 9 ++++----- test/create_mis.cc | 7 +++---- test/curve_to_bezier.cc | 10 +++++----- test/curvetest.cc | 11 +++++------ test/degenerateSurfs.cc | 7 +++---- test/describe.cc | 9 ++++----- test/dg_ma_test.cc | 5 ++--- test/elmBalance.cc | 7 +++---- test/embedded_edges.cc | 7 +++---- test/extrude.cc | 7 +++---- test/fieldReduce.cc | 11 +++++------ test/field_io.cc | 7 +++---- test/fixDisconnected.cc | 7 +++---- test/fixlayer.cc | 5 ++--- test/fixshape.cc | 5 ++--- test/fusion.cc | 9 ++++----- test/fusion2.cc | 11 +++++------ test/fusion3.cc | 5 ++--- test/gap.cc | 7 +++---- test/generate.cc | 17 ++++++++--------- test/ghost.cc | 7 +++---- test/ghostEdge.cc | 5 ++--- test/ghostMPAS.cc | 5 ++--- test/gmsh.cc | 11 +++++------ test/graphdist.cc | 7 +++---- test/hierarchic.cc | 7 +++---- test/highOrderSizeFields.cc | 9 ++++----- test/highOrderSolutionTransfer.cc | 11 +++++------ test/icesheet.cc | 5 ++--- test/inClosureOf_test.cc | 3 +-- test/intrude.cc | 7 +++---- test/loadPart.cc | 8 ++++---- test/ma_insphere.cc | 5 ++--- test/ma_test.cc | 5 ++--- test/ma_test_analytic_model.cc | 5 ++--- test/makeAllCavities.cc | 7 +++---- test/matchedNodeElmReader.cc | 11 +++++------ test/measureAnisoStats.cc | 7 +++---- test/measureIsoStats.cc | 7 +++---- test/mixedNumbering.cc | 5 ++--- test/mkmodel.cc | 7 +++---- test/mktopomodel.cc | 7 +++---- test/modelInfo.cc | 5 ++--- test/moving.cc | 5 ++--- test/nedelecShapes.cc | 7 +++---- test/nektar_align.cc | 5 ++--- test/neper.cc | 7 +++---- test/newdim.cc | 5 ++--- test/osh2smb.cc | 7 +++---- test/ph_adapt.cc | 7 +++---- test/poisson.cc | 7 +++---- test/print_pumipic_partition.cc | 11 +++++------ test/ptnParma.cc | 13 ++++++------- test/pumi.cc | 13 ++++++------- test/pumiLoadMesh.cc | 5 ++--- test/quality.cc | 9 ++++----- test/refine2x.cc | 7 +++---- test/render.cc | 7 +++---- test/renderClass.cc | 7 +++---- test/render_ascii.cc | 7 +++---- test/reorder.cc | 7 +++---- test/repartition.cc | 9 ++++----- test/reposition.cc | 7 +++---- test/residualErrorEstimation_test.cc | 5 ++--- test/rm_extrusion.cc | 13 ++++++------- test/runSimxAnisoAdapt.cc | 13 ++++++------- test/scale.cc | 7 +++---- test/serialize.cc | 7 +++---- test/shapefun.cc | 7 +++---- test/shapefun2.cc | 5 ++--- test/simZBalance.cc | 5 ++--- test/sim_part.cc | 7 +++---- test/smb2osh.cc | 7 +++---- test/snap.cc | 5 ++--- test/split.cc | 13 ++++++------- test/spr_test.cc | 7 +++---- test/swapDoubles.cc | 3 +-- test/test_integrator.cc | 5 ++--- test/test_matrix_grad.cc | 5 ++--- test/test_scaling.cc | 5 ++--- test/test_verify.cc | 5 ++--- test/tetrahedronize.cc | 5 ++--- test/torus_ma_test.cc | 5 ++--- test/ugrid.cc | 14 +++++++------- test/ugridptnstats.cc | 5 ++--- test/uniform.cc | 7 +++---- test/verify.cc | 5 ++--- test/verify_2nd_order_shapes.cc | 7 +++---- test/verify_convert.cc | 9 ++++----- test/visualizeAnisoSizes.cc | 7 +++---- test/viz.cc | 7 +++---- test/vtxBalance.cc | 7 +++---- test/vtxEdgeElmBalance.cc | 7 +++---- test/vtxElmBalance.cc | 7 +++---- test/vtxElmMixedBalance.cc | 5 ++--- test/writeIPFieldTest.cc | 5 ++--- test/writePart.cc | 7 +++---- test/writeVtxPtn.cc | 7 +++---- test/xgc_split.cc | 7 +++---- test/zbalance.cc | 7 +++---- test/zsplit.cc | 13 ++++++------- 125 files changed, 402 insertions(+), 524 deletions(-) diff --git a/test/1d.cc b/test/1d.cc index 0e823f477..654eab63d 100644 --- a/test/1d.cc +++ b/test/1d.cc @@ -6,7 +6,6 @@ #include #include #include -#include void createMesh(gmi_model*& g, apf::Mesh2*& m, int n, pcu::PCU *PCUObj) { @@ -67,14 +66,14 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_model* g; apf::Mesh2* m; int nverts = atoi(argv[1]); PCU_ALWAYS_ASSERT(2 <= nverts); PCU_ALWAYS_ASSERT(nverts <= 1000); - createMesh(g,m,nverts,pcu_obj.get()); + createMesh(g,m,nverts,&pcu_obj); test(m); gmi_write_dmg(g,argv[2]); m->writeNative(argv[3]); diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc index e45ba32e3..ece447aee 100644 --- a/test/H1Shapes.cc +++ b/test/H1Shapes.cc @@ -13,7 +13,6 @@ #include #include #include -#include // User defined vector functions E(x,y,z) of order up to 6 @@ -30,12 +29,12 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -45,12 +44,12 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],&PCUObj); m->verify(); // test fields interpolating a user-defined vector field for (int i = 1; i <= 6; i++) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) lion_oprint(1, "----TESTING VECTOR FIELD OF ORDER %d----\n", i); testH1( m, /* mesh */ @@ -62,7 +61,7 @@ int main(int argc, char** argv) // test fields interpolating a user-defined matrix field for (int i = 1; i <= 6; i++) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) lion_oprint(1, "----TESTING MATRIX FIELD OF ORDER %d----\n", i); testH1( m, /* mesh */ diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc index cbe2a4a12..9f7525769 100644 --- a/test/L2Shapes.cc +++ b/test/L2Shapes.cc @@ -13,7 +13,6 @@ #include #include #include -#include using namespace std; @@ -33,12 +32,12 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -48,7 +47,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],&PCUObj); m->verify(); for (int i = 0; i <= 6; i++) { diff --git a/test/align.cc b/test/align.cc index ed334a7e9..83166344d 100644 --- a/test/align.cc +++ b/test/align.cc @@ -5,7 +5,6 @@ #include #include #include -#include void testTriEdge(pcu::PCU *PCUObj) { @@ -92,12 +91,12 @@ int main() { MPI_Init(0,0); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); - testTriEdge(pcu_obj.get()); - testTetEdge(pcu_obj.get()); - testTetTri(pcu_obj.get()); + testTriEdge(&pcu_obj); + testTetEdge(&pcu_obj); + testTetTri(&pcu_obj); } MPI_Finalize(); } diff --git a/test/aniso_ma_test.cc b/test/aniso_ma_test.cc index 763f72a83..499dbfbf5 100644 --- a/test/aniso_ma_test.cc +++ b/test/aniso_ma_test.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include @@ -47,10 +46,10 @@ int main(int argc, char** argv) bool logInterpolation = atoi(argv[3]) > 0 ? true : false; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); apf::writeVtkFiles("aniso_before",m); AnIso sf(m); diff --git a/test/ansys.cc b/test/ansys.cc index 57d82fbeb..e9573b084 100644 --- a/test/ansys.cc +++ b/test/ansys.cc @@ -4,22 +4,21 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); - apf::Mesh2* m = apf::loadMdsFromANSYS(argv[1], argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsFromANSYS(argv[1], argv[2], &pcu_obj); m->verify(); gmi_write_dmg(m->getModel(), argv[3]); m->writeNative(argv[4]); diff --git a/test/assert_timing.cc b/test/assert_timing.cc index 63e679263..699f0441d 100644 --- a/test/assert_timing.cc +++ b/test/assert_timing.cc @@ -5,7 +5,6 @@ #include #include #include -#include double check_c_assert() { double t0 = pcu::Time(); @@ -32,7 +31,7 @@ int main(int argc, char** argv) { int opt = atoi(argv[1]); MPI_Init(0,0); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); /* i'm avoiding conditionals inside for loops b/c i'm paranoid about the timings even though timings diff --git a/test/balance.cc b/test/balance.cc index cb6b2814c..68efd60e4 100644 --- a/test/balance.cc +++ b/test/balance.cc @@ -5,18 +5,17 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); apf::MeshTag* weights = Parma_WeighByMemory(m); double step = 0.1; int verbose=1; apf::Balancer* balancer = Parma_MakeCentroidDiffuser(m, step, verbose); diff --git a/test/bezierElevation.cc b/test/bezierElevation.cc index 6187fc696..2bb018624 100644 --- a/test/bezierElevation.cc +++ b/test/bezierElevation.cc @@ -10,7 +10,6 @@ #include #include #include -#include /* * This contains all the tests for bezier elevation @@ -400,11 +399,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - testEdgeElevation(pcu_obj.get()); - testTriElevation(pcu_obj.get()); - testTetElevation(pcu_obj.get()); + testEdgeElevation(&pcu_obj); + testTriElevation(&pcu_obj); + testTetElevation(&pcu_obj); } MPI_Finalize(); } diff --git a/test/bezierMesh.cc b/test/bezierMesh.cc index da106938d..fbf317c3f 100644 --- a/test/bezierMesh.cc +++ b/test/bezierMesh.cc @@ -9,7 +9,6 @@ #include #include #include -#include /* this test file contains tests for * a curved 2D mesh @@ -586,11 +585,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - test2D(pcu_obj.get()); - test3DBlended(pcu_obj.get()); - test3DFull(pcu_obj.get()); + test2D(&pcu_obj); + test3DBlended(&pcu_obj); + test3DFull(&pcu_obj); } MPI_Finalize(); } diff --git a/test/bezierMisc.cc b/test/bezierMisc.cc index 4af003d20..2f3b389c2 100644 --- a/test/bezierMisc.cc +++ b/test/bezierMisc.cc @@ -14,7 +14,6 @@ #include #include #include -#include /* This file contains miscellaneous tests relating to ordering, math * and transformation matrices */ @@ -168,7 +167,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); testNodeIndexing(); testMatrixInverse(); diff --git a/test/bezierRefine.cc b/test/bezierRefine.cc index be9a8d81a..f413108b2 100644 --- a/test/bezierRefine.cc +++ b/test/bezierRefine.cc @@ -13,7 +13,6 @@ #include #include -#include // face areas are 1/2 and 19/30 void vert0(double const p[2], double x[3], void*) @@ -312,10 +311,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - test2D(pcu_obj.get()); - test3D(pcu_obj.get()); + test2D(&pcu_obj); + test3D(&pcu_obj); } MPI_Finalize(); } diff --git a/test/bezierShapeEval.cc b/test/bezierShapeEval.cc index db78f5f5f..ba6b5136d 100644 --- a/test/bezierShapeEval.cc +++ b/test/bezierShapeEval.cc @@ -15,7 +15,6 @@ #include #include #include -#include /* This file contains miscellaneous tests relating to bezier shapes and * blended bezier shapes */ @@ -28,10 +27,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 7 ) { - if ( !PCUObj.get()->Self() ) { + if ( !PCUObj.Self() ) { printf("Usage: %s >\n", argv[0]); printf(" can only be 2 (for triangles) and 4 (for tets)\n"); printf(" is the order of bezier\n"); @@ -51,7 +50,7 @@ int main(int argc, char** argv) " must be between -1 and 2!"); apf::MeshEntity* ent = 0; - apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent,PCUObj.get()) : makeOneTetMesh(order,ent,PCUObj.get()); + apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent,&PCUObj) : makeOneTetMesh(order,ent,&PCUObj); double xi0 = atof(argv[4]); double xi1 = atof(argv[5]); diff --git a/test/bezierSubdivision.cc b/test/bezierSubdivision.cc index 111f65f45..308722fe5 100644 --- a/test/bezierSubdivision.cc +++ b/test/bezierSubdivision.cc @@ -10,7 +10,6 @@ #include #include #include -#include /* * This contains all the tests for bezier subdivision @@ -483,12 +482,12 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - testEdgeSubdivision(pcu_obj.get()); - testTriSubdivision1(pcu_obj.get()); - testTriSubdivision4(pcu_obj.get()); - testTetSubdivision1(pcu_obj.get()); + testEdgeSubdivision(&pcu_obj); + testTriSubdivision1(&pcu_obj); + testTriSubdivision4(&pcu_obj); + testTetSubdivision1(&pcu_obj); } MPI_Finalize(); } diff --git a/test/bezierValidity.cc b/test/bezierValidity.cc index b0bf0657a..9f627c3e0 100644 --- a/test/bezierValidity.cc +++ b/test/bezierValidity.cc @@ -13,7 +13,6 @@ #include #include -#include /* This test file uses an alternative and more traditional method to * compute Jacobian differences, using the property of Bezier's that @@ -408,10 +407,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - test2D(pcu_obj.get()); - test3D(pcu_obj.get()); + test2D(&pcu_obj); + test3D(&pcu_obj); } MPI_Finalize(); } diff --git a/test/box.cc b/test/box.cc index fc0999f64..a5ed9b0dc 100644 --- a/test/box.cc +++ b/test/box.cc @@ -6,7 +6,6 @@ #include #include #include -#include namespace { @@ -62,17 +61,17 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - verifyArgs(argc, argv, pcu_obj.get()); - if(pcu_obj.get()->Peers()>1) { - if(pcu_obj.get()->Self()) + verifyArgs(argc, argv, &pcu_obj); + if(pcu_obj.Peers()>1) { + if(pcu_obj.Self()) fprintf(stderr, "%s must be run on a single process!\n", argv[0]); exit(1); } getArgs(argv); gmi_register_mesh(); - apf::Mesh2* m = apf::makeMdsBox(nx,ny,nz,wx,wy,wz,is,pcu_obj.get()); + apf::Mesh2* m = apf::makeMdsBox(nx,ny,nz,wx,wy,wz,is,&pcu_obj); gmi_model* g = m->getModel(); m->verify(); m->writeNative(meshFile); diff --git a/test/capVol.cc b/test/capVol.cc index 25b38091e..ea2fa676f 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -1,6 +1,5 @@ #include #include -#include // Output #include @@ -78,7 +77,7 @@ int main(int argc, char** argv) { // Initialize parallelism. MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); // Initialize logging. lion_set_stdout(stdout); @@ -86,7 +85,7 @@ int main(int argc, char** argv) { // Check arguments or print usage. if (argc < 3) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printUsage(argv[0]); } myExit(EXIT_FAILURE); @@ -174,7 +173,7 @@ int main(int argc, char** argv) { MG_API_CALL(m, compute_adjacency()); // Make APF adapter over Capstone mesh. - ma::Mesh* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); + ma::Mesh* apfCapMesh = apf::createMesh(m, g, &PCUObj); // Choose appropriate size-field. ma::AnisotropicFunction* sf = nullptr; diff --git a/test/cgns.cc b/test/cgns.cc index d8713cc8c..dbce1cc10 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -16,7 +16,6 @@ #include #include #include -#include // https://gist.github.com/bgranzow/98087114166956646da684ed98acab02 apf::MeshTag *create_int_tag(const std::string &name, apf::Mesh2 *m, int dim) @@ -481,13 +480,13 @@ int main(int argc, char **argv) #ifdef HAVE_CGNS MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi_load_pcu(PCUObj.get()); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); lion_set_verbosity(1); bool additionalTests = false; if (argc < 3) { - if (!PCUObj.get()->Self()) + if (!PCUObj.Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -499,7 +498,7 @@ int main(int argc, char **argv) additionalTests = true; else { - if (!PCUObj.get()->Self()) + if (!PCUObj.Self()) printf("Usage: %s additional\n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -508,7 +507,7 @@ int main(int argc, char **argv) } else if (argc > 4) { - if (!PCUObj.get()->Self()) + if (!PCUObj.Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -520,7 +519,7 @@ int main(int argc, char **argv) { apf::CGNSBCMap cgnsBCMap; std::vector> meshData; - cgnsOutputName = doit(cgnsBCMap, argv[1], argv[2], "", additionalTests, additionalTests, meshData, PCUObj.get()); + cgnsOutputName = doit(cgnsBCMap, argv[1], argv[2], "", additionalTests, additionalTests, meshData, &PCUObj); } // Phase 2 if (additionalTests) @@ -543,7 +542,7 @@ int main(int argc, char **argv) meshData.push_back(std::make_pair("CellCenter", "DummyMatrix_1")); apf::CGNSBCMap cgnsBCMap; std::cout << "RE-READING " << std::endl; - doit(cgnsBCMap, cgnsOutputName.c_str(), "tempy.smb", "_reread", false, true, meshData, PCUObj.get()); + doit(cgnsBCMap, cgnsOutputName.c_str(), "tempy.smb", "_reread", false, true, meshData, &PCUObj); } // } diff --git a/test/classifyThenAdapt.cc b/test/classifyThenAdapt.cc index ad8cce792..081ddf5c7 100644 --- a/test/classifyThenAdapt.cc +++ b/test/classifyThenAdapt.cc @@ -6,7 +6,6 @@ #include #include #include -#include #define LEFT_EDGE 0 #define TOP_EDGE 1 @@ -128,11 +127,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); - ma::Mesh* m = createTriMesh(pcu_obj.get()); + ma::Mesh* m = createTriMesh(&pcu_obj); m->verify(); // make sure the mesh is valid! gmi_model* g = m->getModel(); gmi_write_dmg(g,"model.dmg"); diff --git a/test/collapse.cc b/test/collapse.cc index f989c58d3..2ddced964 100644 --- a/test/collapse.cc +++ b/test/collapse.cc @@ -10,7 +10,6 @@ #include #include #include -#include namespace { @@ -47,7 +46,7 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); pcu::Protect(); #ifdef HAVE_SIMMETRIX @@ -56,10 +55,10 @@ int main(int argc, char** argv) { gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,pcu_obj.get()); + getConfig(argc,argv,&pcu_obj); GroupCode code; - code.mesh = apf::loadMdsMesh(modelFile, meshFile, pcu_obj.get()); - apf::Unmodulo outMap(pcu_obj.get()->Self(), pcu_obj.get()->Peers()); + code.mesh = apf::loadMdsMesh(modelFile, meshFile, &pcu_obj); + apf::Unmodulo outMap(pcu_obj.Self(), pcu_obj.Peers()); Parma_ShrinkPartition(code.mesh, partitionFactor, code); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); diff --git a/test/construct.cc b/test/construct.cc index 647994c5e..cfb5597bd 100644 --- a/test/construct.cc +++ b/test/construct.cc @@ -6,14 +6,13 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -23,7 +22,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -31,7 +30,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false, PCUObj.get()); + m = apf::makeEmptyMdsMesh(model, dim, false, &PCUObj); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index 66b8e1ac4..a4d17cf1a 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -8,15 +8,14 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi_load_pcu(pcu_obj.get()); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&pcu_obj); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -26,7 +25,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], &pcu_obj); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -34,7 +33,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false, pcu_obj.get()); + m = apf::makeEmptyMdsMesh(model, dim, false, &pcu_obj); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; diff --git a/test/construct_bottom_up.cc b/test/construct_bottom_up.cc index 4daa384cd..f982dff88 100644 --- a/test/construct_bottom_up.cc +++ b/test/construct_bottom_up.cc @@ -6,7 +6,6 @@ #include #include #include -#include /* This executable demos how to make a PUMI mesh using a bottom up approach. * The info needed to achieve this: @@ -140,10 +139,10 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.get()->Peers() == 1, "Not implemented in parallel!"); + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.Peers() == 1, "Not implemented in parallel!"); if (argc < 2) { printf("USAGE 1 ( no model): %s \n", argv[0]); printf("USAGE 2 (with model): %s #include #include -#include using namespace std; @@ -433,7 +432,7 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); //for fancy BL/extrusion queries @@ -441,12 +440,12 @@ int main(int argc, char** argv) Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv, pcu_obj.get()); + getConfig(argc, argv, &pcu_obj); if( should_log ) Sim_logOn("convert.sim.log"); if (should_attach_order && should_fix_pyramids) { - if (!pcu_obj.get()->Self()) + if (!pcu_obj.Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = false; } @@ -458,7 +457,7 @@ int main(int argc, char** argv) gmi_model* mdl; if( gmi_native_path ) { - if (!pcu_obj.get()->Self()) + if (!pcu_obj.Self()) fprintf(stderr, "loading native model %s\n", gmi_native_path); mdl = gmi_sim_load(gmi_native_path,gmi_path); } else { @@ -487,10 +486,10 @@ int main(int argc, char** argv) double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); double t1 = pcu::Time(); - if(!pcu_obj.get()->Self()) + if(!pcu_obj.Self()) fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, pcu_obj.get()); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, &pcu_obj); addFathersTag(simModel, sim_mesh, simApfMesh, extruRootPath); diff --git a/test/crack_test.cc b/test/crack_test.cc index 3eedce621..cbb7f1ef5 100644 --- a/test/crack_test.cc +++ b/test/crack_test.cc @@ -19,7 +19,6 @@ #include #include #include -#include void bCurver(const char* modelFile, const char* meshFile, @@ -47,10 +46,10 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc < 2) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -70,7 +69,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_model* g = gmi_load(modelFile); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false, PCUObj.get()); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false, &PCUObj); std::vector verts; // vertex coordinates @@ -350,7 +349,7 @@ int main(int argc, char** argv) for (int i = 2; i < 7; i++) { char output[256]; snprintf(output,256,"crack_curved_to_order_%d", i); - bCurver(modelFile, "crack_linear.smb", PCUObj.get(), i, output); + bCurver(modelFile, "crack_linear.smb", &PCUObj, i, output); } #ifdef HAVE_SIMMETRIX diff --git a/test/create_mis.cc b/test/create_mis.cc index 50378c4d6..bedc28f0d 100644 --- a/test/create_mis.cc +++ b/test/create_mis.cc @@ -6,16 +6,15 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -23,7 +22,7 @@ int main(int argc, char** argv) gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); //We will create a field with the coloring as the values on each element apf::Field* coloring = apf::createField(m,"colors",apf::SCALAR, diff --git a/test/curve_to_bezier.cc b/test/curve_to_bezier.cc index 03ffce574..fb9fadc0c 100644 --- a/test/curve_to_bezier.cc +++ b/test/curve_to_bezier.cc @@ -10,27 +10,27 @@ #include #include #include -#include + int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); if ( argc != 5 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } int order = atoi(argv[3]); if(order < 1 || order > 6){ - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Only 1st to 6th order supported\n"); MPI_Finalize(); exit(EXIT_FAILURE); @@ -38,7 +38,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_register_sim(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); crv::BezierCurver bc(m,order,0); bc.run(); diff --git a/test/curvetest.cc b/test/curvetest.cc index 7f7c52223..2b4b51750 100644 --- a/test/curvetest.cc +++ b/test/curvetest.cc @@ -11,7 +11,6 @@ #include #include #include -#include class Linear : public ma::IsotropicFunction { @@ -193,22 +192,22 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); int ne = m->count(1); int nf = m->count(2); m->destroyNative(); apf::destroyMesh(m); - testInterpolating(modelFile,meshFile,ne,nf,PCUObj.get()); - testBezier(modelFile,meshFile,ne,nf,PCUObj.get()); - testGregory(modelFile,meshFile,ne,nf,PCUObj.get()); + testInterpolating(modelFile,meshFile,ne,nf,&PCUObj); + testBezier(modelFile,meshFile,ne,nf,&PCUObj); + testGregory(modelFile,meshFile,ne,nf,&PCUObj); } gmi_sim_stop(); diff --git a/test/degenerateSurfs.cc b/test/degenerateSurfs.cc index 91dd9f94b..71ece5203 100644 --- a/test/degenerateSurfs.cc +++ b/test/degenerateSurfs.cc @@ -12,7 +12,6 @@ #endif #include #include -#include const char* modelFile = 0; const char* meshFile = 0; @@ -38,7 +37,7 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==5); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -48,8 +47,8 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + getConfig(argc,argv,&PCUObj); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); int order = 2; diff --git a/test/describe.cc b/test/describe.cc index 14e6c376d..4ed8b59eb 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -16,7 +16,6 @@ #include #endif #include -#include static void print_stats(const char* name, double value, pcu::PCU *PCUObj) @@ -47,7 +46,7 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -57,10 +56,10 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - print_stats("kernal used before", pcu::GetMem(), pcu_obj.get()); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + print_stats("kernal used before", pcu::GetMem(), &pcu_obj); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); m->verify(); - print_stats("kernel heap", pcu::GetMem(), pcu_obj.get()); + print_stats("kernel heap", pcu::GetMem(), &pcu_obj); Parma_PrintPtnStats(m, ""); list_tags(m); m->destroyNative(); diff --git a/test/dg_ma_test.cc b/test/dg_ma_test.cc index 9ddec672d..5e2b7b50a 100644 --- a/test/dg_ma_test.cc +++ b/test/dg_ma_test.cc @@ -11,7 +11,6 @@ #include #endif #include -#include class Linear : public ma::IsotropicFunction { @@ -42,7 +41,7 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -52,7 +51,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); Linear sf(m); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); diff --git a/test/elmBalance.cc b/test/elmBalance.cc index ab5e806c7..be0d2a2bf 100644 --- a/test/elmBalance.cc +++ b/test/elmBalance.cc @@ -12,7 +12,6 @@ #endif #include #include -#include apf::MeshTag* setWeights(apf::Mesh* m) { apf::MeshIterator* it = m->begin(m->getDimension()); @@ -30,10 +29,10 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -46,7 +45,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); if(!m->getPCU()->Self()) diff --git a/test/embedded_edges.cc b/test/embedded_edges.cc index 73f07230f..fcfa4b565 100644 --- a/test/embedded_edges.cc +++ b/test/embedded_edges.cc @@ -8,7 +8,6 @@ #include #include #include -#include // this test checks that the destruct function works // with meshes that have a lower dimension than the manifold // which tey reside in. E.g. a truss or beam in 3D space @@ -18,7 +17,7 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -29,7 +28,7 @@ int main(int argc, char** argv) int nverts; gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsMesh(model, argv[1], PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(model, argv[1], &PCUObj); apf::deriveMdsModel(m); int dim = m->getDimension(); extractCoords(m, coords, nverts); @@ -41,7 +40,7 @@ int main(int argc, char** argv) std::cout<typeDimension[apf::Mesh::EDGE]< #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); ma::ModelExtrusions extrusions; extrusions.push_back(ma::ModelExtrusion( m->findModelEntity(1, 2), diff --git a/test/fieldReduce.cc b/test/fieldReduce.cc index c9bc5d21c..4bc94ac65 100644 --- a/test/fieldReduce.cc +++ b/test/fieldReduce.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace { @@ -168,10 +167,10 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); bool failflag = false; { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - MPI_Comm_rank(PCUObj.get()->GetMPIComm(), &myrank); - MPI_Comm_size(PCUObj.get()->GetMPIComm(), &commsize); + MPI_Comm_rank(PCUObj.GetMPIComm(), &myrank); + MPI_Comm_size(PCUObj.GetMPIComm(), &commsize); #ifdef HAVE_SIMMETRIX MS_init(); SimModel_start(); @@ -180,11 +179,11 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); + getConfig(argc,argv,&PCUObj); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; - m = apf::loadMdsMesh(g, meshFile, PCUObj.get()); + m = apf::loadMdsMesh(g, meshFile, &PCUObj); for (int i=0; i < 3; ++i) diff --git a/test/field_io.cc b/test/field_io.cc index 36f39be10..44c2c8305 100644 --- a/test/field_io.cc +++ b/test/field_io.cc @@ -5,18 +5,17 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); { - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); apf::Field* f = apf::createLagrangeField(m, "foo", apf::VECTOR, 1); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* vert; @@ -29,7 +28,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); } { - apf::Mesh2* m = apf::loadMdsMesh(argv[1], "tmp.smb", PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], "tmp.smb", &PCUObj); apf::Field* f = m->findField("foo"); PCU_ALWAYS_ASSERT(f); PCU_ALWAYS_ASSERT(apf::VECTOR == apf::getValueType(f)); diff --git a/test/fixDisconnected.cc b/test/fixDisconnected.cc index 492755efa..5901e84b5 100644 --- a/test/fixDisconnected.cc +++ b/test/fixDisconnected.cc @@ -6,23 +6,22 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); Parma_PrintPtnStats(m, "initial"); Parma_ProcessDisconnectedParts(m); Parma_PrintPtnStats(m, "final"); diff --git a/test/fixlayer.cc b/test/fixlayer.cc index dd5fd26d9..f018e51c0 100644 --- a/test/fixlayer.cc +++ b/test/fixlayer.cc @@ -10,14 +10,13 @@ #include #endif #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -27,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); + ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldCleanupLayer = true; ma::adapt(in); diff --git a/test/fixshape.cc b/test/fixshape.cc index 048bc0d61..fc32ab791 100644 --- a/test/fixshape.cc +++ b/test/fixshape.cc @@ -10,14 +10,13 @@ #include #endif #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -27,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldFixShape = true; ma::adapt(in); diff --git a/test/fusion.cc b/test/fusion.cc index ba91827da..f9fc9e216 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -7,7 +7,6 @@ #include #include #include -#include double const a_param = 0.2; double const b_param = 1.0; @@ -157,16 +156,16 @@ int main( int argc, char* argv[]) PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); GroupCode code; code.model = makeModel(); code.meshFile = argv[1]; - apf::Unmodulo outMap(pcu_obj.get()->Self(), 2); - Parma_SplitPartition(nullptr, 2, code, pcu_obj.get()); + apf::Unmodulo outMap(pcu_obj.Self(), 2); + Parma_SplitPartition(nullptr, 2, code, &pcu_obj); //Have to call switchPCU here because the mesh needed to be made inside GroupCode run() //and inside parma_group.cc runInGroups() we set code.PCUObj to groupedPCU - code.mesh->switchPCU(pcu_obj.get()); + code.mesh->switchPCU(&pcu_obj); apf::remapPartition(code.mesh, outMap); code.mesh->verify(); code.mesh->destroyNative(); diff --git a/test/fusion2.cc b/test/fusion2.cc index 33fb8a407..d252af990 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -6,7 +6,6 @@ #include #include #include -#include static apf::Mesh2* makeEmptyMesh(pcu::PCU *PCUObj) { @@ -140,15 +139,15 @@ int main( int argc, char* argv[]) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 2); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 2); gmi_register_null(); GroupCode code; - code.mesh = makeEmptyMesh(PCUObj.get()); + code.mesh = makeEmptyMesh(&PCUObj); int const groupSize = 1; - apf::Unmodulo outMap(PCUObj.get()->Self(), groupSize); - Parma_SplitPartition(code.mesh, groupSize, code, PCUObj.get()); + apf::Unmodulo outMap(PCUObj.Self(), groupSize); + Parma_SplitPartition(code.mesh, groupSize, code, &PCUObj); globalCode(code.mesh); } MPI_Finalize(); diff --git a/test/fusion3.cc b/test/fusion3.cc index c44ec2a13..ee3e562c1 100644 --- a/test/fusion3.cc +++ b/test/fusion3.cc @@ -9,7 +9,6 @@ #include #include #include -#include using std::vector; class Expression @@ -284,11 +283,11 @@ int main(int argc, char * argv[]) PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_model* model = makeModel(); gmi_write_dmg(model, "made.dmg"); - apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1], PCUObj.get()); + apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1], &PCUObj); mesh->verify(); Vortex sfv(mesh, center, modelLen); const ma::Input* in = ma::configure(mesh,&sfv); diff --git a/test/gap.cc b/test/gap.cc index 11fe1c72a..36554ffd7 100644 --- a/test/gap.cc +++ b/test/gap.cc @@ -12,7 +12,6 @@ #endif #include #include -#include namespace { apf::MeshTag* setWeights(apf::Mesh* m) { @@ -31,10 +30,10 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc == 5); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -48,7 +47,7 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setWeights(m); int verbose = 2; // set to 1 to silence the 'endStep' stats diff --git a/test/generate.cc b/test/generate.cc index 1a8b31ffb..3f0ad6768 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -21,7 +21,6 @@ #include #include #include -#include #include //cout #include //option parser @@ -323,28 +322,28 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - globalPCU = PCUObj.get(); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + globalPCU = &PCUObj; lion_set_verbosity(1); pcu::Protect(); - getConfig(argc,argv,PCUObj.get()); + getConfig(argc,argv,&PCUObj); if (should_attach_order && should_fix_pyramids) { - if (!PCUObj.get()->Self()) + if (!PCUObj.Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = 0; } simStart(); - pNativeModel nm = loadNativeModel(PCUObj.get()); + pNativeModel nm = loadNativeModel(&PCUObj); pGModel simModel = GM_load(modelFile.c_str(), nm, NULL); const double t0 = MPI_Wtime(); - pParMesh sim_mesh = generate(simModel, caseName, PCUObj.get()); + pParMesh sim_mesh = generate(simModel, caseName, &PCUObj); const double t1 = MPI_Wtime(); - if(!PCUObj.get()->Self()) + if(!PCUObj.Self()) printf("Mesh generated in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, PCUObj.get()); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, &PCUObj); if (should_attach_order) attachOrder(simApfMesh); gmi_register_sim(); diff --git a/test/ghost.cc b/test/ghost.cc index 6b1b59b5b..190feb5eb 100644 --- a/test/ghost.cc +++ b/test/ghost.cc @@ -6,7 +6,6 @@ #include #include #include -#include namespace { const char* modelFile = 0; @@ -79,11 +78,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + getConfig(argc,argv,&PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); apf::MeshTag* weights = applyFun3dWeight(m); runParma(m,weights); m->destroyTag(weights); diff --git a/test/ghostEdge.cc b/test/ghostEdge.cc index c91ef9929..6f4d3ae40 100644 --- a/test/ghostEdge.cc +++ b/test/ghostEdge.cc @@ -5,7 +5,6 @@ #include #include #include -#include namespace { const char* modelFile = 0; @@ -52,11 +51,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); runParma(m); m->writeNative(argv[3]); freeMesh(m); diff --git a/test/ghostMPAS.cc b/test/ghostMPAS.cc index e908b4528..8af944c6a 100644 --- a/test/ghostMPAS.cc +++ b/test/ghostMPAS.cc @@ -5,7 +5,6 @@ #include #include #include -#include namespace { const char* modelFile = 0; @@ -54,11 +53,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); runParma(m); m->writeNative(argv[3]); freeMesh(m); diff --git a/test/gmsh.cc b/test/gmsh.cc index 999f6215e..2161ec764 100644 --- a/test/gmsh.cc +++ b/test/gmsh.cc @@ -6,16 +6,15 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n" "The input .msh and output .smb file names are required. \n" "If 'none' is specified as the input model file name then \n" @@ -40,15 +39,15 @@ int main(int argc, char** argv) apf::Mesh2* m = NULL; if (gmshVersion == 2) { if (model.compare("none") == 0) { - m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str(), PCUObj.get()); + m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str(), &PCUObj); apf::deriveMdsModel(m); gmi_write_dmg(m->getModel(),outModel.c_str()); } else { - m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str(), PCUObj.get()); + m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str(), &PCUObj); } } else if (gmshVersion == 4) { if (model.compare("none") == 0) { - m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str(), PCUObj.get()); + m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str(), &PCUObj); } } m->verify(); diff --git a/test/graphdist.cc b/test/graphdist.cc index 7834a93c7..3d9e96041 100644 --- a/test/graphdist.cc +++ b/test/graphdist.cc @@ -6,22 +6,21 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); dsp::Boundary moving; moving.insert(m->findModelEntity(2, 57)); moving.insert(m->findModelEntity(2, 62)); diff --git a/test/hierarchic.cc b/test/hierarchic.cc index 2dbec0134..26bccc7de 100644 --- a/test/hierarchic.cc +++ b/test/hierarchic.cc @@ -8,7 +8,6 @@ #include #include #include -#include namespace { @@ -195,11 +194,11 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(! pcu_obj.get()->Self()); + PCU_ALWAYS_ASSERT(! pcu_obj.Self()); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); apf::reorderMdsMesh(m); m->verify(); int p_order = atoi(argv[3]); diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc index 7e4806299..db6a71ffb 100644 --- a/test/highOrderSizeFields.cc +++ b/test/highOrderSizeFields.cc @@ -19,7 +19,6 @@ #include #include #include -#include void computeSizesFrames( @@ -38,7 +37,7 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX @@ -51,16 +50,16 @@ int main(int argc, char** argv) gmi_register_mesh(); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr <<"usage: " << argv[0] << " \n"; return EXIT_FAILURE; } for (int order = 3; order < 5; order++) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) lion_oprint(1, "Testing aniso adapt w/ sizefield order %d\n", order); - testAdapt(argv[1], argv[2], order, PCUObj.get()); + testAdapt(argv[1], argv[2], order, &PCUObj); } } diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index 430bae19a..57eb318d5 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -16,7 +16,6 @@ #include #include #include -#include void E_exact(const apf::Vector3& x, apf::Vector3& value, int p); @@ -44,7 +43,7 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -64,23 +63,23 @@ int main(int argc, char** argv) */ // linear adapt - testCurveAdapt(modelFile, meshFile, PCUObj.get(), + testCurveAdapt(modelFile, meshFile, &PCUObj, 1 /*mesh_order*/, 2 /*exact_order*/, 2 /*field_order*/); // quadratic adapts - testCurveAdapt(modelFile, meshFile, PCUObj.get(), + testCurveAdapt(modelFile, meshFile, &PCUObj, 2 /*mesh_order*/, 2 /*exact_order*/, 4 /*field_order*/); - testCurveAdapt(modelFile, meshFile, PCUObj.get(), + testCurveAdapt(modelFile, meshFile, &PCUObj, 2 /*mesh_order*/, 3 /*exact_order*/, 6 /*field_order*/); // cubic adapt - testCurveAdapt(modelFile, meshFile, PCUObj.get(), + testCurveAdapt(modelFile, meshFile, &PCUObj, 3 /*mesh_order*/, 2 /*exact_order*/, 6 /*field_order*/); diff --git a/test/icesheet.cc b/test/icesheet.cc index fe627f31a..a05b4061d 100644 --- a/test/icesheet.cc +++ b/test/icesheet.cc @@ -8,7 +8,6 @@ #include #include #include -#include /* tags on vertices */ #define INTERIORTAG 0 @@ -484,7 +483,7 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -495,7 +494,7 @@ int main(int argc, char** argv) readMesh(argv[1],m); const int dim = 3; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false, PCUObj.get()); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false, &PCUObj); apf::GlobalToVert outMap; apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap); delete [] m.elements; diff --git a/test/inClosureOf_test.cc b/test/inClosureOf_test.cc index a7b60aff7..f76448979 100644 --- a/test/inClosureOf_test.cc +++ b/test/inClosureOf_test.cc @@ -12,7 +12,6 @@ #endif #include #include -#include int main(int argc, char** argv) { @@ -21,7 +20,7 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); diff --git a/test/intrude.cc b/test/intrude.cc index e79a302b8..bf1fbe332 100644 --- a/test/intrude.cc +++ b/test/intrude.cc @@ -6,22 +6,21 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); ma::ModelExtrusions extrusions; extrusions.push_back(ma::ModelExtrusion( m->findModelEntity(1, 2), diff --git a/test/loadPart.cc b/test/loadPart.cc index fb6395316..32d4067f7 100644 --- a/test/loadPart.cc +++ b/test/loadPart.cc @@ -12,11 +12,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 1); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Load a single part from a partitioned mesh and " "write it as a serial part.\n" "Usage: %s \n", argv[0]); @@ -26,7 +26,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_register_mesh(); gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsPart(g, argv[1], PCUObj.get()); + apf::Mesh2* m = apf::loadMdsPart(g, argv[1], &PCUObj); apf::deriveMdsModel(m); gmi_write_dmg(g, argv[3]); m->writeNative(argv[2]); diff --git a/test/ma_insphere.cc b/test/ma_insphere.cc index 6319e94af..4b990a705 100644 --- a/test/ma_insphere.cc +++ b/test/ma_insphere.cc @@ -4,7 +4,6 @@ #include #include #include -#include int main(int argc, char** argv) { @@ -31,7 +30,7 @@ int main(int argc, char** argv) // Test insphere (create a mesh with one tet) { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); apf::Vector3 a(0, 0, 0); apf::Vector3 b(-6, 0, 0); @@ -40,7 +39,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, 3, true, PCUObj.get()); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, 3, true, &PCUObj); apf::ModelEntity* m = mesh->findModelEntity(0, 0); apf::MeshEntity* v[4]; for (int i=0; i<4; i++) { diff --git a/test/ma_test.cc b/test/ma_test.cc index 4d60d00be..86100546b 100644 --- a/test/ma_test.cc +++ b/test/ma_test.cc @@ -11,7 +11,6 @@ #include #endif #include -#include class Linear : public ma::IsotropicFunction { @@ -46,7 +45,7 @@ int main(int argc, char** argv) const double adaptRefineFactor = (argc==5) ? atoi(argv[4]) : 3; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -56,7 +55,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); Linear sf(m,adaptRefineFactor); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); diff --git a/test/ma_test_analytic_model.cc b/test/ma_test_analytic_model.cc index b12d01317..1a03f233d 100644 --- a/test/ma_test_analytic_model.cc +++ b/test/ma_test_analytic_model.cc @@ -13,7 +13,6 @@ #include #include -#include const double pi = 3.14159265359; @@ -172,10 +171,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - apf::Mesh2* m = createSphereMesh(PCUObj.get()); + apf::Mesh2* m = createSphereMesh(&PCUObj); m->verify(); apf::writeVtkFiles("initial_mesh_on_analytic_model", m); diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc index b5c2935af..df5692c48 100644 --- a/test/makeAllCavities.cc +++ b/test/makeAllCavities.cc @@ -12,7 +12,6 @@ #include #include #include -#include #ifdef HAVE_SIMMETRIX #include #include @@ -64,8 +63,8 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - if (PCUObj.get()->Peers() > 1) { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + if (PCUObj.Peers() > 1) { printf("%s should only be used for serial (single part) meshes!\n", argv[0]); printf("use the serialize utility to get a serial mesh, and retry!\n"); MPI_Finalize(); @@ -112,7 +111,7 @@ int main(int argc, char** argv) // load the mesh and check if the tag exists on the mesh - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); if (mode.compare(std::string("aa")) == 0) tag = tagMesh(m, -1, 1); diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index f0f8b3a4e..a135f0703 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -18,7 +18,6 @@ #include #include #include -#include /* from https://github.com/SCOREC/core/issues/205 0=fully interior of the volume @@ -746,11 +745,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); int noVerify=0; // maintain default of verifying if not explicitly requesting it off if( argc < 11 ) { - if( !PCUObj.get()->Self() ) { + if( !PCUObj.Self() ) { printf("Usage: %s " " " " " @@ -773,17 +772,17 @@ int main(int argc, char** argv) double t0 = pcu::Time(); MeshInfo m; - readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m,PCUObj.get()); + readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m,&PCUObj); bool isMatched = true; if( !strcmp(argv[3], "NULL") ) isMatched = false; - if(!PCUObj.get()->Self()) + if(!PCUObj.Self()) fprintf(stderr, "isMatched %d\n", isMatched); gmi_model* model = gmi_load(argv[1]); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched, PCUObj.get()); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched, &PCUObj); apf::GlobalToVert outMap; for( size_t i=0; i< m.elements.size(); i++) { apf::assemble(mesh, m.elements[i], m.numElms[i], m.elementType[i], outMap); diff --git a/test/measureAnisoStats.cc b/test/measureAnisoStats.cc index d00d3777b..c748303b2 100644 --- a/test/measureAnisoStats.cc +++ b/test/measureAnisoStats.cc @@ -21,7 +21,6 @@ #include #include #include -#include // === includes for safe_mkdir === #include @@ -44,10 +43,10 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc < 5) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE1: %s " "\n", argv[0]); printf("USAGE2: %s " @@ -78,7 +77,7 @@ int main(int argc, char** argv) safe_mkdir(inPrefix); - getStats(".null", meshFile, sizeName, frameName, inPrefix, PCUObj.get()); + getStats(".null", meshFile, sizeName, frameName, inPrefix, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/test/measureIsoStats.cc b/test/measureIsoStats.cc index 87b60f760..4ebacec5e 100644 --- a/test/measureIsoStats.cc +++ b/test/measureIsoStats.cc @@ -22,7 +22,6 @@ #include #include #include -#include // === includes for safe_mkdir === #include @@ -45,10 +44,10 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc < 4) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE1: %s " "\n", argv[0]); printf("USAGE2: %s " @@ -78,7 +77,7 @@ int main(int argc, char** argv) safe_mkdir(inPrefix); - getStats(".null", meshFile, sizeName, inPrefix, PCUObj.get()); + getStats(".null", meshFile, sizeName, inPrefix, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/test/mixedNumbering.cc b/test/mixedNumbering.cc index 8eb4f7f11..9017b454d 100644 --- a/test/mixedNumbering.cc +++ b/test/mixedNumbering.cc @@ -8,7 +8,6 @@ #include #include #include -#include static void test_numbering(apf::Mesh* m) { apf::FieldShape* S2 = apf::getSerendipity(); @@ -50,10 +49,10 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); apf::reorderMdsMesh(m); test_numbering(m); write_output(m, argv[3]); diff --git a/test/mkmodel.cc b/test/mkmodel.cc index 41bf26743..3d5e063f3 100644 --- a/test/mkmodel.cc +++ b/test/mkmodel.cc @@ -6,23 +6,22 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 3 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Create a discrete geometric model from a mesh\n" "Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); - apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], &pcu_obj); gmi_model* g = m->getModel(); gmi_write_dmg(g, argv[2]); m->destroyNative(); diff --git a/test/mktopomodel.cc b/test/mktopomodel.cc index 522474994..e0b64d360 100644 --- a/test/mktopomodel.cc +++ b/test/mktopomodel.cc @@ -6,16 +6,15 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 3 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Create a topological geometric model from a mesh\n" "Usage: %s \n", argv[0]); MPI_Finalize(); @@ -23,7 +22,7 @@ int main(int argc, char** argv) } gmi_register_null(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], &pcu_obj); apf::deriveMdsModel(m); gmi_model* g = m->getModel(); gmi_write_dmg(g, argv[2]); diff --git a/test/modelInfo.cc b/test/modelInfo.cc index 2fa9865b7..f35450f7d 100644 --- a/test/modelInfo.cc +++ b/test/modelInfo.cc @@ -10,17 +10,16 @@ #endif #include //exit and exit_failure #include -#include #include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 2 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); diff --git a/test/moving.cc b/test/moving.cc index 36ab1d29a..7e297fe32 100644 --- a/test/moving.cc +++ b/test/moving.cc @@ -7,7 +7,6 @@ #include #include #include -#include static void writeStep(apf::Mesh* m, int i) { @@ -21,14 +20,14 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 3 ) { fprintf(stderr, "Usage: %s \n", argv[0]); return 0; } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); dsp::Boundary moving; moving.insert(m->findModelEntity(2, 57)); moving.insert(m->findModelEntity(2, 62)); diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc index 6c642a448..534f4d04b 100644 --- a/test/nedelecShapes.cc +++ b/test/nedelecShapes.cc @@ -13,7 +13,6 @@ #include #include #include -#include using namespace std; @@ -29,12 +28,12 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(0); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -44,7 +43,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],&PCUObj); m->verify(); diff --git a/test/nektar_align.cc b/test/nektar_align.cc index 0abb124ab..62010da87 100644 --- a/test/nektar_align.cc +++ b/test/nektar_align.cc @@ -13,7 +13,6 @@ #include #endif #include -#include namespace apf { /* the more dangerous a function is, @@ -111,7 +110,7 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -121,7 +120,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); alignForNektar(m); m->writeNative(argv[3]); m->destroyNative(); diff --git a/test/neper.cc b/test/neper.cc index 738f78d5d..8be481036 100644 --- a/test/neper.cc +++ b/test/neper.cc @@ -4,22 +4,21 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2], &pcu_obj); m->verify(); m->writeNative(argv[3]); m->destroyNative(); diff --git a/test/newdim.cc b/test/newdim.cc index 4956a3d84..29b68d160 100644 --- a/test/newdim.cc +++ b/test/newdim.cc @@ -3,17 +3,16 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj.get()); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, &PCUObj); apf::Vector3 points[4] = { apf::Vector3(0,0,0), apf::Vector3(1,0,0), diff --git a/test/osh2smb.cc b/test/osh2smb.cc index af0d5225f..c45eb6777 100644 --- a/test/osh2smb.cc +++ b/test/osh2smb.cc @@ -6,7 +6,6 @@ #include #include #include -#include #include @@ -17,10 +16,10 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 4) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { std::cout << "\n"; std::cout << "usage: osh2smb in.osh in.dmg out.smb\n"; std::cout << " or: osh2smb (usage)\n"; @@ -35,7 +34,7 @@ int main(int argc, char** argv) { auto lib = Omega_h::Library(&argc, &argv); Omega_h::Mesh om(&lib); Omega_h::binary::read(argv[1], lib.world(), &om); - apf::Mesh2* am = apf::makeEmptyMdsMesh(model, om.dim(), false, PCUObj.get()); + apf::Mesh2* am = apf::makeEmptyMdsMesh(model, om.dim(), false, &PCUObj); apf::from_omega_h(am, &om); am->writeNative(argv[3]); am->destroyNative(); diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index 7593ddcb5..72a9a7d20 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -18,7 +18,6 @@ #include #endif #include -#include static bool overwriteAPFCoord(apf::Mesh2* m) { apf::Field* f = m->findField("motion_coords"); @@ -49,7 +48,7 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -60,10 +59,10 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); /* load model, mesh and configure input */ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); ph::Input in; - in.load("adapt.inp", PCUObj.get()); + in.load("adapt.inp", &PCUObj); in.openfile_read = openfile_read; /* attach solution and other fields */ ph::readAndAttachFields(in,m); diff --git a/test/poisson.cc b/test/poisson.cc index df8b6421c..1d97d048c 100644 --- a/test/poisson.cc +++ b/test/poisson.cc @@ -9,7 +9,6 @@ #include #include #include -#include namespace { @@ -253,12 +252,12 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); MPI_Init(&argc, &argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(! pcu_obj.get()->Self()); + PCU_ALWAYS_ASSERT(! pcu_obj.Self()); int dim = atoi(argv[1]); int p = atoi(argv[2]); - test(dim, p, pcu_obj.get()); + test(dim, p, &pcu_obj); } MPI_Finalize(); } diff --git a/test/print_pumipic_partition.cc b/test/print_pumipic_partition.cc index ff445e4ad..371a43614 100644 --- a/test/print_pumipic_partition.cc +++ b/test/print_pumipic_partition.cc @@ -4,7 +4,6 @@ #include #include #include -#include #ifdef HAVE_SIMMETRIX #include @@ -21,16 +20,16 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 && argc != 6) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } - if (pcu_obj.get()->Peers() > 1) { - if ( !pcu_obj.get()->Self() ) + if (pcu_obj.Peers() > 1) { + if ( !pcu_obj.Self() ) printf("This tool must be run in serial.\n"); MPI_Finalize(); exit(EXIT_FAILURE); @@ -45,7 +44,7 @@ int main(int argc, char** argv) gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); int num_ranks = atoi(argv[3]); //Partition the mesh (Taken from zsplit.cc) diff --git a/test/ptnParma.cc b/test/ptnParma.cc index d1e767746..d43a22a9f 100644 --- a/test/ptnParma.cc +++ b/test/ptnParma.cc @@ -14,7 +14,6 @@ #include #endif #include -#include namespace { @@ -165,9 +164,9 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - if( !PCUObj.get()->Self() ) + if( !PCUObj.Self() ) lion_oprint(1, "PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); #ifdef HAVE_SIMMETRIX MS_init(); @@ -178,11 +177,11 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); lion_set_verbosity(1); - getConfig(argc,argv,PCUObj.get()); - if (PCUObj.get()->Self() % partitionFactor) - mymain(false, PCUObj.get()); + getConfig(argc,argv,&PCUObj); + if (PCUObj.Self() % partitionFactor) + mymain(false, &PCUObj); else - mymain(true, PCUObj.get()); + mymain(true, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); diff --git a/test/pumi.cc b/test/pumi.cc index 48adfb108..59aea11fd 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -12,7 +12,6 @@ #include #include #include -#include const char* modelFile = 0; const char* meshFile = 0; @@ -89,25 +88,25 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi_load_pcu(PCUObj.get()); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); lion_set_verbosity(1); pumi_printSys(); #if 0 int i, processid = getpid(); - if (!PCUObj.get()->Self()) + if (!PCUObj.Self()) { - std::cout<<"Proc "<Self()<<">> pid "<> pid "<>i; } else - std::cout<<"Proc "<Self()<<">> pid "<> pid "< int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi_load_pcu(PCUObj.get()); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); pGeom g = pumi_geom_load(argv[1], "mesh"); pMesh m = pumi_mesh_load(g, argv[2], 1); pumi_mesh_delete(m); diff --git a/test/quality.cc b/test/quality.cc index 90fc9586b..17f1097a1 100644 --- a/test/quality.cc +++ b/test/quality.cc @@ -7,7 +7,6 @@ #include #include #include -#include namespace { @@ -92,13 +91,13 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + getConfig(argc,argv,&PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); processMesh(m); - printDiagnostics(PCUObj.get()); + printDiagnostics(&PCUObj); m->destroyNative(); apf::destroyMesh(m); } diff --git a/test/refine2x.cc b/test/refine2x.cc index a8afcde19..c8e80b14d 100644 --- a/test/refine2x.cc +++ b/test/refine2x.cc @@ -14,7 +14,6 @@ #include #include #include -#include class AnisotropicX: public ma::AnisotropicFunction { public: @@ -86,10 +85,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 5) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -102,7 +101,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); AnisotropicX* ansx = new AnisotropicX(m, atoi(argv[3])); ma::Input* in = ma::makeAdvanced(ma::configure(m, ansx)); #ifdef PUMI_HAS_ZOLTAN diff --git a/test/render.cc b/test/render.cc index 354b09ded..52ef4e3ab 100644 --- a/test/render.cc +++ b/test/render.cc @@ -13,16 +13,15 @@ #endif #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -41,7 +40,7 @@ int main(int argc, char** argv) // does not fail when the input mesh is curved! crv::getBezier(2); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); std::string name = m->getShape()->getName(); int order = m->getShape()->getOrder(); diff --git a/test/renderClass.cc b/test/renderClass.cc index 85c1a7701..2d110c685 100644 --- a/test/renderClass.cc +++ b/test/renderClass.cc @@ -12,7 +12,6 @@ #include #endif #include -#include static void number_dim(apf::Mesh* m, apf::FieldShape* shape, int dim, std::string const& prefix) { std::string name = prefix + "_class_dim"; @@ -35,10 +34,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (!(argc == 4 || argc == 5)) { - if ( !PCUObj.get()->Self() ) { + if ( !PCUObj.Self() ) { printf("Usage: %s \n", argv[0]); printf(" %s \n", argv[0]); } @@ -55,7 +54,7 @@ int main(int argc, char** argv) gmi_register_mesh(); char const* modelpath = argv[1]; char const* meshpath = argv[2]; - apf::Mesh2* m = apf::loadMdsMesh(modelpath, meshpath, PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelpath, meshpath, &PCUObj); int dim; char const* vtkpath; if (argc == 5) { diff --git a/test/render_ascii.cc b/test/render_ascii.cc index c9e9d00ac..2d748f1ee 100644 --- a/test/render_ascii.cc +++ b/test/render_ascii.cc @@ -10,16 +10,15 @@ #include #endif #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -32,7 +31,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::writeASCIIVtkFiles(argv[3], m); m->destroyNative(); apf::destroyMesh(m); diff --git a/test/reorder.cc b/test/reorder.cc index e7c014fea..94d6670bb 100644 --- a/test/reorder.cc +++ b/test/reorder.cc @@ -12,15 +12,14 @@ #include #endif #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -34,7 +33,7 @@ int main(int argc, char** argv) { #endif gmi_register_null(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::MeshTag* order = Parma_BfsReorder(m); apf::reorderMdsMesh(m, order); m->writeNative(argv[3]); diff --git a/test/repartition.cc b/test/repartition.cc index 08bf1c3bb..07b85e6fd 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -9,7 +9,6 @@ #include #include #include -#include namespace { @@ -99,18 +98,18 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto expanded_pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU expanded_pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv,expanded_pcu_obj.get()); + getConfig(argc,argv,&expanded_pcu_obj); gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = 0; - CreateGroupCommResult result = createGroupComm(expanded_pcu_obj.get()); + CreateGroupCommResult result = createGroupComm(&expanded_pcu_obj); if (result.isOriginal) m = apf::loadMdsMesh(g, meshFile, result.group_pcu_obj); - m = apf::expandMdsMesh(m, g, inputPartCount, expanded_pcu_obj.get()); + m = apf::expandMdsMesh(m, g, inputPartCount, &expanded_pcu_obj); balance(m); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); diff --git a/test/reposition.cc b/test/reposition.cc index 326411b1d..c3d4b47ce 100644 --- a/test/reposition.cc +++ b/test/reposition.cc @@ -4,18 +4,17 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #if 0 gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj.get()); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, &PCUObj); apf::Vector3 vx[4] = {apf::Vector3(0,0,0), apf::Vector3(1,0,0), @@ -28,7 +27,7 @@ int main(int argc, char** argv) apf::MeshEntity* v = tv[0]; #else gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* v; while ((v = m->iterate(it))) { diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc index d4c662056..a34934e5a 100644 --- a/test/residualErrorEstimation_test.cc +++ b/test/residualErrorEstimation_test.cc @@ -12,7 +12,6 @@ #include #endif #include -#include void E_exact(const apf::Vector3 &x, apf::Vector3& E); double computeElementExactError(apf::Mesh* mesh, apf::MeshEntity* e, @@ -28,7 +27,7 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -38,7 +37,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); kappa = freq * M_PI; diff --git a/test/rm_extrusion.cc b/test/rm_extrusion.cc index 13995dbe0..f415fd11f 100644 --- a/test/rm_extrusion.cc +++ b/test/rm_extrusion.cc @@ -18,7 +18,6 @@ #include #include #include -#include @@ -94,7 +93,7 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); @@ -102,7 +101,7 @@ int main(int argc, char** argv) Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv, PCUObj.get()); + getConfig(argc, argv, &PCUObj); if( should_log ) Sim_logOn("rm_extrusion.sim.log"); @@ -117,21 +116,21 @@ int main(int argc, char** argv) else mdl = gmi_load(gmi_path); pGModel simModel = gmi_export_sim(mdl); - if(!PCUObj.get()->Self()) + if(!PCUObj.Self()) fprintf(stderr, "Read model\n"); double t0 = pcu::Time(); pMesh sim_mesh = M_load(sms_path, simModel, progress); - if(!PCUObj.get()->Self()) + if(!PCUObj.Self()) fprintf(stderr, "Read mesh\n"); M_removeSurfaceExtrusionConstraints(sim_mesh, NULL); - if(!PCUObj.get()->Self()) + if(!PCUObj.Self()) fprintf(stderr, "Removed surface extrusion constraints\n"); M_write(sim_mesh, smsNew_path, 0, progress); double t1 = pcu::Time(); - if(!PCUObj.get()->Self()) + if(!PCUObj.Self()) fprintf(stderr, "read the mesh, removed the face extrusion attributes, and wrote the mesh %f seconds\n", t1-t0); M_release(sim_mesh); diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc index eae558fd6..81896ad28 100644 --- a/test/runSimxAnisoAdapt.cc +++ b/test/runSimxAnisoAdapt.cc @@ -18,7 +18,6 @@ #include #include #include -#include using namespace std; @@ -77,14 +76,14 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); // Call before calling Sim_readLicenseFile Sim_readLicenseFile(0); SimDiscrete_start(0); // initialize GeomSim Discrete library if (argc < 7) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s " " \n", argv[0]); } @@ -95,7 +94,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_register_null(); - PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.get()->Peers() == 1, + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.Peers() == 1, "This utility only works for serial meshes!"); const char* inputPumiModel = argv[1]; @@ -119,7 +118,7 @@ int main(int argc, char** argv) snprintf(outImprovedSimxMesh, 256, "%s_adapted_improved.sms", prefix); snprintf(outImprovedPumiMesh, 256, "%s_adapted_improved.smb", prefix); - apf::Mesh2* m = apf::loadMdsMesh(inputPumiModel, inputPumiMesh, PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(inputPumiModel, inputPumiMesh, &PCUObj); char message[512]; // first find the sizes field @@ -194,7 +193,7 @@ int main(int argc, char** argv) printf("%s\n", outAdaptedSimxMesh); printf("%s\n", outAdaptedPumiMesh); M_write(simxMesh,outAdaptedSimxMesh, 0,0); // write out the initial mesh data - apf::Mesh2* m2 = convertToPumi(simxMesh, dim, sizeName, frameName, PCUObj.get()); + apf::Mesh2* m2 = convertToPumi(simxMesh, dim, sizeName, frameName, &PCUObj); m2->writeNative(outAdaptedPumiMesh); printf("===DONE===\n"); @@ -208,7 +207,7 @@ int main(int argc, char** argv) printf("%s\n", outImprovedSimxMesh); printf("%s\n", outImprovedPumiMesh); M_write(simxMesh,outImprovedSimxMesh, 0,0); // write out the initial mesh data - apf::Mesh2* m3 = convertToPumi(simxMesh, dim, sizeName, frameName, PCUObj.get()); + apf::Mesh2* m3 = convertToPumi(simxMesh, dim, sizeName, frameName, &PCUObj); m3->writeNative(outImprovedPumiMesh); printf("===DONE===\n"); diff --git a/test/scale.cc b/test/scale.cc index 40f153035..d321a066a 100644 --- a/test/scale.cc +++ b/test/scale.cc @@ -5,7 +5,6 @@ #include #include #include -#include namespace { @@ -56,10 +55,10 @@ static void scale_mesh(apf::Mesh2* m, Scale const& s) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - if (argc != 8) print_usage(argv, PCUObj.get()); + if (argc != 8) print_usage(argv, &PCUObj); const char* gfile = argv[1]; const char* mfile = argv[2]; const char* ofile = argv[3]; @@ -68,7 +67,7 @@ int main(int argc, char** argv) { scale.y = atof(argv[5]); scale.z = atof(argv[6]); scale.s = atof(argv[7]); - apf::Mesh2* m = apf::loadMdsMesh(gfile, mfile, PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(gfile, mfile, &PCUObj); m->verify(); scale_mesh(m, scale); m->verify(); diff --git a/test/serialize.cc b/test/serialize.cc index 2ba122835..74b7c5c47 100644 --- a/test/serialize.cc +++ b/test/serialize.cc @@ -12,7 +12,6 @@ #endif #include #include -#include struct GroupCode : public Parma_GroupCode { @@ -28,10 +27,10 @@ int main( int argc, char* argv[]) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -47,7 +46,7 @@ int main( int argc, char* argv[]) gmi_register_null(); crv::getBezier(2);//hack to make sure curved meshes can be serialized! GroupCode code; - code.mesh = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); + code.mesh = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); code.meshFile = argv[3]; apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); Parma_ShrinkPartition(code.mesh, atoi(argv[4]), code); diff --git a/test/shapefun.cc b/test/shapefun.cc index bbf7fd336..7c02b1d2b 100644 --- a/test/shapefun.cc +++ b/test/shapefun.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include @@ -245,7 +244,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); testP1LineNodeValues(); @@ -260,8 +259,8 @@ int main(int argc, char** argv) testPrismNodeValues(); testPyramidNodeValues(); testQuadrilateralNodeValues(); - testPrismVolume(PCUObj.get()); - testPyramidVolume(PCUObj.get()); + testPrismVolume(&PCUObj); + testPyramidVolume(&PCUObj); } MPI_Finalize(); } diff --git a/test/shapefun2.cc b/test/shapefun2.cc index 9fde54b5d..5922c36e7 100644 --- a/test/shapefun2.cc +++ b/test/shapefun2.cc @@ -6,7 +6,6 @@ #include #include #include -#include namespace test { @@ -172,7 +171,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); @@ -181,7 +180,7 @@ int main(int argc, char** argv) crv::getBezier(1),crv::getBezier(2)}; for (int i = 0; i < 4; ++i) - test::checkFieldShape(fs[i], PCUObj.get()); + test::checkFieldShape(fs[i], &PCUObj); } MPI_Finalize(); diff --git a/test/simZBalance.cc b/test/simZBalance.cc index 7676d487b..910da6a34 100644 --- a/test/simZBalance.cc +++ b/test/simZBalance.cc @@ -10,7 +10,6 @@ #include #include #include -#include static void load_balance(apf::Mesh2* m) { @@ -27,7 +26,7 @@ static void load_balance(apf::Mesh2* m) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); Sim_readLicenseFile(NULL); gmi_sim_start(); @@ -40,7 +39,7 @@ int main(int argc, char** argv) { gmi_model* apf_model = gmi_sim_load(0, smd_file); pGModel sim_model = gmi_export_sim(apf_model); pParMesh sim_mesh = PM_load(sms_file, sim_model, NULL); - apf::Mesh2* apf_mesh = apf::createMesh(sim_mesh, PCUObj.get()); + apf::Mesh2* apf_mesh = apf::createMesh(sim_mesh, &PCUObj); //apf_mesh->verify(); <- this calls Simmetrix's verify function apf::verify(apf_mesh); load_balance(apf_mesh); diff --git a/test/sim_part.cc b/test/sim_part.cc index c43b8fbee..83906b02f 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -46,7 +46,6 @@ #include #include #include -#include using namespace std; @@ -69,7 +68,7 @@ int main(int argc, char **argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); pcu::Protect(); // Initialize PartitionedMesh - this should be the first Simmetrix call @@ -109,7 +108,7 @@ int main(int argc, char **argv) outmeshFilename = tmp.c_str(); /* print message */ - if (PCUObj.get()->Self()==0) { + if (PCUObj.Self()==0) { cout<Self()==0) { + if (PCUObj.Self()==0) { cout<<"**********************************"< #include #include -#include #include @@ -17,10 +16,10 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 4) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { std::cout << "\n"; std::cout << "usage: smb2osh in.dmg in.smb out.osh\n"; std::cout << " or: smb2osh (usage)\n"; @@ -30,7 +29,7 @@ int main(int argc, char** argv) { } gmi_register_mesh(); gmi_register_null(); - apf::Mesh2* am = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); + apf::Mesh2* am = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); { auto lib = Omega_h::Library(&argc, &argv); Omega_h::Mesh om(&lib); diff --git a/test/snap.cc b/test/snap.cc index 825645c46..13953fa2b 100644 --- a/test/snap.cc +++ b/test/snap.cc @@ -7,21 +7,20 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); const ma::Input* in = ma::configureIdentity(m); ma::adapt(in); m->writeNative(argv[3]); diff --git a/test/split.cc b/test/split.cc index 48d1ee7ba..6dbcc4c18 100644 --- a/test/split.cc +++ b/test/split.cc @@ -12,7 +12,6 @@ #endif #include #include -#include namespace { @@ -69,7 +68,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -79,22 +78,22 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - bool isOriginal = ((PCUObj.get()->Self() % partitionFactor) == 0); + getConfig(argc,argv,&PCUObj); + bool isOriginal = ((PCUObj.Self() % partitionFactor) == 0); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj.get()); + pcu::PCU *groupedPCUObj = getGroupedPCU(&PCUObj); if (isOriginal) { m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh //on the globalPCU - if(m != nullptr) m->switchPCU(PCUObj.get()); + if(m != nullptr) m->switchPCU(&PCUObj); delete groupedPCUObj; - m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); + m = repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); diff --git a/test/spr_test.cc b/test/spr_test.cc index 7aa3dfb95..69276fdb2 100644 --- a/test/spr_test.cc +++ b/test/spr_test.cc @@ -6,7 +6,6 @@ #include #include #include -#include int main(int argc, char** argv) { @@ -20,10 +19,10 @@ int main(int argc, char** argv) const int order = atoi(argv[4]); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile, PCUObj.get()); + apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile, &PCUObj); if (mesh->findTag("coordinates_edg")) mesh->changeShape(apf::getSerendipity(), false); apf::Field* f = @@ -31,7 +30,7 @@ int main(int argc, char** argv) apf::Field* eps = spr::getGradIPField(f, "eps", order); apf::destroyField(f); double adaptRatio = 0.1; - apf::Field* sizef = spr::getSPRSizeField(eps,adaptRatio,PCUObj.get()); + apf::Field* sizef = spr::getSPRSizeField(eps,adaptRatio,&PCUObj); apf::destroyField(eps); writeVtkFiles(outFile,mesh); apf::destroyField(sizef); diff --git a/test/swapDoubles.cc b/test/swapDoubles.cc index 6019ef4fc..4b9e8a9f1 100644 --- a/test/swapDoubles.cc +++ b/test/swapDoubles.cc @@ -3,12 +3,11 @@ #include //PCU_ALWAYS_ASSERT #include //iota #include //cerr -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); const size_t n = 2; double *d_orig = new double[n]; std::iota(d_orig,d_orig+n,0); diff --git a/test/test_integrator.cc b/test/test_integrator.cc index ee715ace5..41f1cd320 100644 --- a/test/test_integrator.cc +++ b/test/test_integrator.cc @@ -6,7 +6,6 @@ #include #include #include -#include class CountIntegrator : public apf::Integrator { protected: @@ -25,13 +24,13 @@ class CountIntegrator : public apf::Integrator { int main(int argc, char ** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); // argument should be model, mesh PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); gmi_register_null(); - apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); + apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); CountIntegrator * countInt = new CountIntegrator(); // test integration over implicitly defined mesh dimension countInt->process(mesh); diff --git a/test/test_matrix_grad.cc b/test/test_matrix_grad.cc index cc9bbf805..80d702927 100644 --- a/test/test_matrix_grad.cc +++ b/test/test_matrix_grad.cc @@ -7,7 +7,6 @@ #include #include #include -#include /* * This test sets a nodal point matrix on a linear mesh. It then computes the gradient @@ -95,10 +94,10 @@ int main(int argc, char* argv[]) } MPI_Init(&argc, &argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); + apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); int order=1; apf::Field* nodal_matrix_fld = apf::createLagrangeField(mesh, "matrix", apf::MATRIX, order); apf::Field* matrix_deriv = apf::createPackedField(mesh, "matrix_deriv", 27, apf::getIPShape(3,order)); diff --git a/test/test_scaling.cc b/test/test_scaling.cc index da18142b0..f727a902b 100644 --- a/test/test_scaling.cc +++ b/test/test_scaling.cc @@ -8,17 +8,16 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); apf::Field* identity_size = samSz::isoSize(m); double scaling_factor = sam::getIsoLengthScalar(identity_size, m->count(m->getDimension())); diff --git a/test/test_verify.cc b/test/test_verify.cc index 75e4d7b13..9802b09ad 100644 --- a/test/test_verify.cc +++ b/test/test_verify.cc @@ -7,17 +7,16 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::DynamicArray fields(10); apf::FieldShape* shapes[10] = { apf::getLagrange(1), diff --git a/test/tetrahedronize.cc b/test/tetrahedronize.cc index b1375de6b..ee0555833 100644 --- a/test/tetrahedronize.cc +++ b/test/tetrahedronize.cc @@ -10,14 +10,13 @@ #include #endif #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -27,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldTurnLayerToTets = true; ma::adapt(in); diff --git a/test/torus_ma_test.cc b/test/torus_ma_test.cc index 137ffbfe5..5c0799848 100644 --- a/test/torus_ma_test.cc +++ b/test/torus_ma_test.cc @@ -5,7 +5,6 @@ #include #include #include -#include class CylindricalShock : public ma::AnisotropicFunction { @@ -49,10 +48,10 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); apf::writeVtkFiles("torus_before",m); CylindricalShock sf(m); diff --git a/test/ugrid.cc b/test/ugrid.cc index 25f1cd00a..ab35ff931 100644 --- a/test/ugrid.cc +++ b/test/ugrid.cc @@ -32,22 +32,22 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); const int partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj.get()->Peers()); - bool isOriginal = ((PCUObj.get()->Self() % partitionFactor) == 0); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj.Peers()); + bool isOriginal = ((PCUObj.Self() % partitionFactor) == 0); gmi_model* g = gmi_load(".null"); apf::Mesh2* m = 0; apf::Migration* plan = 0; - pcu::PCU *groupedPCUObj = getGroupedPCU(partitionFactor, PCUObj.get()); + pcu::PCU *groupedPCUObj = getGroupedPCU(partitionFactor, &PCUObj); if (isOriginal) { m = apf::loadMdsFromUgrid(g, argv[1], groupedPCUObj); apf::deriveMdsModel(m); @@ -56,9 +56,9 @@ int main(int argc, char** argv) } //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh //on the globalPCU - if(m != nullptr) m->switchPCU(PCUObj.get()); + if(m != nullptr) m->switchPCU(&PCUObj); delete groupedPCUObj; - m = repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); + m = repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); gmi_write_dmg(g,argv[2]); m->writeNative(argv[3]); diff --git a/test/ugridptnstats.cc b/test/ugridptnstats.cc index 9c79bbb44..c9a67dac4 100644 --- a/test/ugridptnstats.cc +++ b/test/ugridptnstats.cc @@ -5,7 +5,6 @@ #include #include #include -#include const double vtxw = 1.0; const double edgew = 1.0; @@ -21,7 +20,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); PCU_ALWAYS_ASSERT( 3 == argc ); @@ -29,7 +28,7 @@ int main(int argc, char** argv) const char* ptnfile = argv[2]; gmi_register_null(); gmi_model* g = gmi_load(".null"); - apf::printUgridPtnStats(g,ugridfile,ptnfile,weights,PCUObj.get()); + apf::printUgridPtnStats(g,ugridfile,ptnfile,weights,&PCUObj); } MPI_Finalize(); } diff --git a/test/uniform.cc b/test/uniform.cc index f18b211cb..2d45b167b 100644 --- a/test/uniform.cc +++ b/test/uniform.cc @@ -11,7 +11,6 @@ #endif #include #include -#include const char* modelFile = 0; const char* meshFile = 0; @@ -35,7 +34,7 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -45,8 +44,8 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + getConfig(argc,argv,&PCUObj); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 1)); if (in->shouldSnap) { in->shouldSnap = false; diff --git a/test/verify.cc b/test/verify.cc index 1343d443a..76f94e055 100644 --- a/test/verify.cc +++ b/test/verify.cc @@ -10,14 +10,13 @@ #include #endif #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -27,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); m->verify(); m->destroyNative(); apf::destroyMesh(m); diff --git a/test/verify_2nd_order_shapes.cc b/test/verify_2nd_order_shapes.cc index b4143ed07..acfc64f48 100644 --- a/test/verify_2nd_order_shapes.cc +++ b/test/verify_2nd_order_shapes.cc @@ -14,15 +14,14 @@ #endif #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 2 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -37,7 +36,7 @@ int main(int argc, char** argv) { gmi_register_null(); gmi_register_mesh(); gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[1],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[1],&PCUObj); int dim = m->getDimension(); diff --git a/test/verify_convert.cc b/test/verify_convert.cc index 50136fd38..f312d202b 100644 --- a/test/verify_convert.cc +++ b/test/verify_convert.cc @@ -11,7 +11,6 @@ #include #include #include -#include apf::Mesh2* createEmptyMesh(pcu::PCU *PCUObj) { @@ -45,13 +44,13 @@ int main(int argc, char* argv[]) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); // create meshes and write data to one of them - apf::Mesh* m1 = createMesh(PCUObj.get()); - apf::Mesh2* m2 = createEmptyMesh(PCUObj.get()); + apf::Mesh* m1 = createMesh(&PCUObj); + apf::Mesh2* m2 = createEmptyMesh(&PCUObj); // create field on m1 apf::Field* f = apf::createLagrangeField(m1, "field1", apf::SCALAR, 1); apf::Function* func = new twox(f); @@ -139,7 +138,7 @@ int main(int argc, char* argv[]) m2->end(it); // check that not transfering Fields/Numberings/Tags also works - apf::Mesh2* m3 = createEmptyMesh(PCUObj.get()); + apf::Mesh2* m3 = createEmptyMesh(&PCUObj); apf::convert(m1, m3, NULL, NULL, false); m3->verify(); diff --git a/test/visualizeAnisoSizes.cc b/test/visualizeAnisoSizes.cc index 972b2eb60..f27b25b7c 100644 --- a/test/visualizeAnisoSizes.cc +++ b/test/visualizeAnisoSizes.cc @@ -17,7 +17,6 @@ #include #include #include -#include // === includes for safe_mkdir === #include @@ -45,10 +44,10 @@ int main(int argc, char** argv) MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc < 8) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE1: %s " " \n", argv[0]); printf("USAGE2: %s " @@ -79,7 +78,7 @@ int main(int argc, char** argv) int sampleSize[2] = {atoi(argv[5]),atoi(argv[6])}; double scale = atof(argv[7]); safe_mkdir(inPrefix); - visualizeSizeField(".null", meshFile, sizeName, frameName, sampleSize, scale, inPrefix, PCUObj.get()); + visualizeSizeField(".null", meshFile, sizeName, frameName, sampleSize, scale, inPrefix, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/test/viz.cc b/test/viz.cc index 6f523d44a..cf99e58fa 100644 --- a/test/viz.cc +++ b/test/viz.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include "../viz/viz.h" namespace { @@ -38,17 +37,17 @@ int main(int argc, char** argv) MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided); PCU_ALWAYS_ASSERT(provided==MPI_THREAD_MULTIPLE); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); Visualization v; char output[128]; - snprintf(output,128,"%d",PCUObj.get()->Self()); + snprintf(output,128,"%d",PCUObj.Self()); std::string part_num(output); apf::MeshIterator* itr; diff --git a/test/vtxBalance.cc b/test/vtxBalance.cc index 6029c2cf9..163330f39 100644 --- a/test/vtxBalance.cc +++ b/test/vtxBalance.cc @@ -12,7 +12,6 @@ #endif #include #include -#include namespace { apf::MeshTag* setVtxWeights(apf::Mesh* m) { @@ -32,10 +31,10 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -49,7 +48,7 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setVtxWeights(m); const double step = 0.5; const int verbose = 1; diff --git a/test/vtxEdgeElmBalance.cc b/test/vtxEdgeElmBalance.cc index 7cce097b1..b79239277 100644 --- a/test/vtxEdgeElmBalance.cc +++ b/test/vtxEdgeElmBalance.cc @@ -12,7 +12,6 @@ #endif #include #include -#include namespace { void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim, double w=1.0) { @@ -47,10 +46,10 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc == 6); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 6 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -65,7 +64,7 @@ int main(int argc, char** argv) gmi_register_mesh(); //load model and mesh double targetImb = atof(argv[5]); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::MeshTag* weights = setWeights(m,atof(argv[4])); Parma_PrintWeightedPtnStats(m, weights, "initial"); const double step = 0.5; const int verbose = 1; diff --git a/test/vtxElmBalance.cc b/test/vtxElmBalance.cc index 88f52ce29..12a47e39c 100644 --- a/test/vtxElmBalance.cc +++ b/test/vtxElmBalance.cc @@ -6,7 +6,6 @@ #include #include #include -#include namespace { void setWeight(apf::Mesh* m, apf::MeshTag* tag, int dim) { @@ -36,17 +35,17 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setWeights(m); const double step = 0.5; const int verbose = 2; diff --git a/test/vtxElmMixedBalance.cc b/test/vtxElmMixedBalance.cc index a033c91cd..3aa55fe99 100644 --- a/test/vtxElmMixedBalance.cc +++ b/test/vtxElmMixedBalance.cc @@ -4,7 +4,6 @@ #include #include #include -#include #ifdef HAVE_SIMMETRIX #include #include @@ -19,7 +18,7 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -29,7 +28,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); ma::localizeLayerStacks(m); m->verify(); diff --git a/test/writeIPFieldTest.cc b/test/writeIPFieldTest.cc index 37a62a12b..8a0f60e01 100644 --- a/test/writeIPFieldTest.cc +++ b/test/writeIPFieldTest.cc @@ -4,17 +4,16 @@ #include #include #include -#include int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); m->verify(); apf::createIPField(m, "Cauchy_Stress", apf::MATRIX, 1); apf::writeVtkFiles("mesh_with_IPField", m); diff --git a/test/writePart.cc b/test/writePart.cc index 2aaa9f78b..4d631ec0e 100644 --- a/test/writePart.cc +++ b/test/writePart.cc @@ -4,22 +4,21 @@ #include #include #include -#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !pcu_obj.get()->Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); if (m->getPCU()->Self() == atoi(argv[3])) apf::writeMdsPart(m, argv[4]); m->destroyNative(); diff --git a/test/writeVtxPtn.cc b/test/writeVtxPtn.cc index 58adae5f2..600519842 100644 --- a/test/writeVtxPtn.cc +++ b/test/writeVtxPtn.cc @@ -6,7 +6,6 @@ #include #include #include -#include namespace { const char* modelFile = 0; @@ -35,11 +34,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj.get()); + getConfig(argc,argv,&PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); } diff --git a/test/xgc_split.cc b/test/xgc_split.cc index a7223d25b..c359c7447 100644 --- a/test/xgc_split.cc +++ b/test/xgc_split.cc @@ -6,7 +6,6 @@ #include #include #include -#include #include "apfMDS.h" #include "apfShape.h" @@ -57,9 +56,9 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - pumi_load_pcu(PCUObj.get()); - getConfig(argc,argv,PCUObj.get()); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); + getConfig(argc,argv,&PCUObj); pGeom g = pumi_geom_load(modelFile); pMesh m; diff --git a/test/zbalance.cc b/test/zbalance.cc index 85a49a934..7409e5291 100644 --- a/test/zbalance.cc +++ b/test/zbalance.cc @@ -8,7 +8,6 @@ #include #include #include // exit and EXIT_FAILURE -#include #ifdef HAVE_SIMMETRIX #include #include @@ -21,10 +20,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCUObj.get()->Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -39,7 +38,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],PCUObj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::MeshTag* weights = Parma_WeighByMemory(m); apf::Balancer* balancer = makeZoltanBalancer(m, apf::GRAPH, apf::REPARTITION); balancer->balance(weights, 1.10); diff --git a/test/zsplit.cc b/test/zsplit.cc index 8011fba86..547201a9a 100644 --- a/test/zsplit.cc +++ b/test/zsplit.cc @@ -13,7 +13,6 @@ #include #include #include -#include namespace { @@ -71,7 +70,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -81,22 +80,22 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,PCUObj.get()); - bool isOriginal = ((PCUObj.get()->Self() % partitionFactor) == 0); + getConfig(argc,argv,&PCUObj); + bool isOriginal = ((PCUObj.Self() % partitionFactor) == 0); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj.get()); + pcu::PCU *groupedPCUObj = getGroupedPCU(&PCUObj); if (isOriginal) { m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh //on the globalPCU - if(m != nullptr) m->switchPCU(PCUObj.get()); + if(m != nullptr) m->switchPCU(&PCUObj); delete groupedPCUObj; - m = apf::repeatMdsMesh(m, g, plan, partitionFactor, PCUObj.get()); + m = apf::repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); From d666a1ee48ba57e60d351778c963431f52e3ca97 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Tue, 27 Aug 2024 11:51:06 -0400 Subject: [PATCH 138/141] Removed uses of unique_ptr in other folders besides test --- capstone_clis/capStone2VTK.cc | 7 +++--- capstone_clis/capStoneAnalyzeArea.cc | 7 +++--- capstone_clis/capStoneAnalyzeEdgeLengths.cc | 7 +++--- capstone_clis/capStoneAnisoAdapt.cc | 7 +++--- capstone_clis/capStoneAnisoAdaptWing.cc | 5 ++-- capstone_clis/capStoneAttachSolution.cc | 7 +++--- capstone_clis/capStoneAttachSolution2.cc | 5 ++-- .../capStoneAttachSolutionStrandOnly.cc | 7 +++--- capstone_clis/capStoneAttachSolutionUni.cc | 7 +++--- capstone_clis/capStoneCheckParametrization.cc | 5 ++-- capstone_clis/capStoneCurve.cc | 3 +-- capstone_clis/capStoneGeomTest.cc | 9 ++++---- capstone_clis/capStoneIsoAdaptB737.cc | 7 +++--- .../maximalIndependentSet/test/testMIS.cc | 21 ++++++++--------- phasta/adaptLvlSet_loop.cc | 23 +++++++++---------- phasta/chef.cc | 9 ++++---- phasta/chefStream.cc | 19 ++++++++------- phasta/condense.cc | 9 ++++---- phasta/cut_interface.cc | 7 +++--- phasta/migrate_interface.cc | 7 +++--- phasta/ph_convert.cc | 11 ++++----- phasta/readUrPrep.cc | 7 +++--- phasta/threshold.cc | 5 ++-- 23 files changed, 89 insertions(+), 112 deletions(-) diff --git a/capstone_clis/capStone2VTK.cc b/capstone_clis/capStone2VTK.cc index a552797a3..c6a8656c1 100644 --- a/capstone_clis/capStone2VTK.cc +++ b/capstone_clis/capStone2VTK.cc @@ -21,7 +21,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -43,10 +42,10 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -133,7 +132,7 @@ int main(int argc, char** argv) gmi_register_cap(); // convert the mesh to apf/mds mesh - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); apf::writeVtkFiles(folderName, mesh); diff --git a/capstone_clis/capStoneAnalyzeArea.cc b/capstone_clis/capStoneAnalyzeArea.cc index c535c0616..6dd3332af 100644 --- a/capstone_clis/capStoneAnalyzeArea.cc +++ b/capstone_clis/capStoneAnalyzeArea.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -50,12 +49,12 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); double initialTime = pcu::Time(); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -145,7 +144,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // add size distribution based on area diff --git a/capstone_clis/capStoneAnalyzeEdgeLengths.cc b/capstone_clis/capStoneAnalyzeEdgeLengths.cc index 7e4881038..df2cb297a 100644 --- a/capstone_clis/capStoneAnalyzeEdgeLengths.cc +++ b/capstone_clis/capStoneAnalyzeEdgeLengths.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -51,12 +50,12 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); double initialTime = pcu::Time(); if (argc != 3) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -146,7 +145,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // add size distribution based on area diff --git a/capstone_clis/capStoneAnisoAdapt.cc b/capstone_clis/capStoneAnisoAdapt.cc index 2e7329c52..da6011294 100644 --- a/capstone_clis/capStoneAnisoAdapt.cc +++ b/capstone_clis/capStoneAnisoAdapt.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -51,10 +50,10 @@ int main(int argc, char** argv) /* INIT CALLS */ MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc < 3) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); printf("Size-fields:\n"); printf("%d, for uniform anisotropic size-field\n", 1); @@ -112,7 +111,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); diff --git a/capstone_clis/capStoneAnisoAdaptWing.cc b/capstone_clis/capStoneAnisoAdaptWing.cc index 92d481459..02167ab21 100644 --- a/capstone_clis/capStoneAnisoAdaptWing.cc +++ b/capstone_clis/capStoneAnisoAdaptWing.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -98,7 +97,7 @@ int main(int argc, char** argv) auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc < 2) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -151,7 +150,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); diff --git a/capstone_clis/capStoneAttachSolution.cc b/capstone_clis/capStoneAttachSolution.cc index 4203d66c9..e951631ae 100644 --- a/capstone_clis/capStoneAttachSolution.cc +++ b/capstone_clis/capStoneAttachSolution.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -600,12 +599,12 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); double initialTime = pcu::Time(); if (argc != 9) { - if(0==PCUObj.get()->Self()) { + if(0==PCUObj.Self()) { std::cerr << "usage: " << argv[0] << " \n"; std::cerr << "*target field(s) is a bit string to select which field(s) are used for error estimation\n"; @@ -713,7 +712,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // make the volume mesh (this one is MDS underneath) diff --git a/capstone_clis/capStoneAttachSolution2.cc b/capstone_clis/capStoneAttachSolution2.cc index 694882496..7d8a4bc89 100644 --- a/capstone_clis/capStoneAttachSolution2.cc +++ b/capstone_clis/capStoneAttachSolution2.cc @@ -20,7 +20,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -500,7 +499,7 @@ int main(int argc, char** argv) double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -599,7 +598,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // remove unused verts diff --git a/capstone_clis/capStoneAttachSolutionStrandOnly.cc b/capstone_clis/capStoneAttachSolutionStrandOnly.cc index 3a758183d..bbd55f42a 100644 --- a/capstone_clis/capStoneAttachSolutionStrandOnly.cc +++ b/capstone_clis/capStoneAttachSolutionStrandOnly.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -522,12 +521,12 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCUObj.get()->Self()) { + if(0==PCUObj.Self()) { std::cerr << "usage: " << argv[0] << " \n"; std::cerr << "*target field(s) is a bit string to select which field(s) are used for error estimation\n"; @@ -558,7 +557,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // make the volume mesh (this one is MDS underneath) diff --git a/capstone_clis/capStoneAttachSolutionUni.cc b/capstone_clis/capStoneAttachSolutionUni.cc index 3a6effd40..8fadfd695 100644 --- a/capstone_clis/capStoneAttachSolutionUni.cc +++ b/capstone_clis/capStoneAttachSolutionUni.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -499,12 +498,12 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -596,7 +595,7 @@ int main(int argc, char** argv) printf("---- CapStone Mesh Loaded. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); //adapt the mesh ma::Input* in; diff --git a/capstone_clis/capStoneCheckParametrization.cc b/capstone_clis/capStoneCheckParametrization.cc index 5273f708b..3ed3adf03 100644 --- a/capstone_clis/capStoneCheckParametrization.cc +++ b/capstone_clis/capStoneCheckParametrization.cc @@ -7,7 +7,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -30,10 +29,10 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc != 2) { - if(0==PCUObj.get()->Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; diff --git a/capstone_clis/capStoneCurve.cc b/capstone_clis/capStoneCurve.cc index 17e62261b..147cc007b 100644 --- a/capstone_clis/capStoneCurve.cc +++ b/capstone_clis/capStoneCurve.cc @@ -13,7 +13,6 @@ #include #include #include -#include // === includes for safe_mkdir === #include @@ -162,7 +161,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* writeVtk(cs, "before.vtk"); */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); apf::MeshEntity* ent; diff --git a/capstone_clis/capStoneGeomTest.cc b/capstone_clis/capStoneGeomTest.cc index 13cc05dd9..9d750523b 100644 --- a/capstone_clis/capStoneGeomTest.cc +++ b/capstone_clis/capStoneGeomTest.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include "CapstoneModule.h" @@ -117,7 +116,7 @@ int main(int argc, char** argv) gmi_register_cap(); - apf::Mesh2* mesh0 = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh0 = apf::createMesh(m,g,&PCUObj); apf::writeVtkFiles("mesh_no_param", mesh0); gmi_model* model = gmi_import_cap(g); @@ -138,7 +137,7 @@ int main(int argc, char** argv) while( (ge = gmi_next(model, gi)) ){ std::stringstream name_str; name_str << "face_" << gmi_tag(model, ge) << "_mesh"; - visualizeFace(model, ge, 100, 100, name_str.str().c_str(), PCUObj.get()); + visualizeFace(model, ge, 100, 100, name_str.str().c_str(), &PCUObj); } gmi_end(model, gi); @@ -146,7 +145,7 @@ int main(int argc, char** argv) printf("creating mesh with param field\n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj.get()); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); apf::Field* pf = apf::createFieldOn(mesh, "param_field", apf::VECTOR); apf::Field* idf = apf::createFieldOn(mesh, "id", apf::SCALAR); apf::MeshEntity* e; @@ -272,7 +271,7 @@ void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* // make the vertexes and set the coordinates using the array std::vector vs; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, PCUObj.get()); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, &PCUObj); for (size_t i = 0; i < ps.size(); i++) { ma::Entity* vert = mesh->createVert(0); mesh->setPoint(vert, 0, ps[i]); diff --git a/capstone_clis/capStoneIsoAdaptB737.cc b/capstone_clis/capStoneIsoAdaptB737.cc index 7ee4aa361..b52d2b4d3 100644 --- a/capstone_clis/capStoneIsoAdaptB737.cc +++ b/capstone_clis/capStoneIsoAdaptB737.cc @@ -14,7 +14,6 @@ #include #include #include -#include #include "CapstoneModule.h" /* #include "CreateMG_Framework_Core.h" */ @@ -219,10 +218,10 @@ int main(int argc, char** argv) /* INIT CALLS */ MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc < 2) { - if (PCUObj.get()->Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -275,7 +274,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g, PCUObj.get()); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); diff --git a/parma/diffMC/maximalIndependentSet/test/testMIS.cc b/parma/diffMC/maximalIndependentSet/test/testMIS.cc index 5fc93af91..1c1c5f4a6 100644 --- a/parma/diffMC/maximalIndependentSet/test/testMIS.cc +++ b/parma/diffMC/maximalIndependentSet/test/testMIS.cc @@ -7,7 +7,6 @@ #include #include #include "parma_commons.h" -#include using std::set; using std::vector; @@ -712,10 +711,10 @@ bool getBool(int& isDebug) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); pcu::Protect(); - int rank = PCUObj.get()->Self(); - int commSize = PCUObj.get()->Peers(); + int rank = PCUObj.Self(); + int commSize = PCUObj.Peers(); int opt; int debugMode = 0; @@ -747,7 +746,7 @@ int main(int argc, char** argv) { testNum = broadcastInt(testNum); - randNumSeed = broadcastInt(randNumSeed) + PCUObj.get()->Self(); + randNumSeed = broadcastInt(randNumSeed) + PCUObj.Self(); bool predefinedRandNums = getBool(iPredefinedRandNums); mis_init(randNumSeed, getBool(debugMode)); @@ -764,22 +763,22 @@ int main(int argc, char** argv) { return 0; break; case 0: - ierr = test_4partsA(rank, commSize, predefinedRandNums, PCUObj.get()); + ierr = test_4partsA(rank, commSize, predefinedRandNums, &PCUObj); break; case 1: - ierr = test_4partsB(rank, commSize, predefinedRandNums, PCUObj.get()); + ierr = test_4partsB(rank, commSize, predefinedRandNums, &PCUObj); break; case 2: - ierr = test_4partsC(rank, commSize, predefinedRandNums, PCUObj.get()); + ierr = test_4partsC(rank, commSize, predefinedRandNums, &PCUObj); break; case 3: - ierr = test_StarA(rank, commSize, PCUObj.get()); + ierr = test_StarA(rank, commSize, &PCUObj); break; case 4: - ierr = test_2dStencil(rank, commSize, PCUObj.get()); + ierr = test_2dStencil(rank, commSize, &PCUObj); break; case 5: - ierr = test_2dStencil(rank,commSize,PCUObj.get(),false,true); + ierr = test_2dStencil(rank,commSize,&PCUObj,false,true); break; } diff --git a/phasta/adaptLvlSet_loop.cc b/phasta/adaptLvlSet_loop.cc index 042c46b46..c5ac347a6 100644 --- a/phasta/adaptLvlSet_loop.cc +++ b/phasta/adaptLvlSet_loop.cc @@ -19,7 +19,6 @@ #endif #include #include -#include namespace { void freeMesh(apf::Mesh* m) { @@ -39,7 +38,7 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); @@ -49,23 +48,23 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(PCUObj.get()); + GRStream* grs = makeGRStream(&PCUObj); ph::Input ctrl; - ctrl.load("adaptLvlSet.inp", PCUObj.get()); + ctrl.load("adaptLvlSet.inp", &PCUObj); //preprocess (define bubbles) - chef::cook(g,m,ctrl,grs,PCUObj.get()); - RStream* rs = makeRStream(PCUObj.get()); - attachRStream(grs,rs,PCUObj.get()); + chef::cook(g,m,ctrl,grs,&PCUObj); + RStream* rs = makeRStream(&PCUObj); + attachRStream(grs,rs,&PCUObj); reconfigureChef(ctrl); // attach the solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs,grs,PCUObj.get()); - attachRStream(grs,rs,PCUObj.get()); + chef::cook(g,m,ctrl,rs,grs,&PCUObj); + attachRStream(grs,rs,&PCUObj); // attach a solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs,PCUObj.get()); - destroyGRStream(grs,PCUObj.get()); - destroyRStream(rs,PCUObj.get()); + chef::cook(g,m,ctrl,rs,&PCUObj); + destroyGRStream(grs,&PCUObj); + destroyRStream(rs,&PCUObj); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/phasta/chef.cc b/phasta/chef.cc index 40b03453e..1fd034d8c 100644 --- a/phasta/chef.cc +++ b/phasta/chef.cc @@ -3,7 +3,6 @@ #include #include #include -#include #ifdef HAVE_SIMMETRIX #include #include @@ -32,10 +31,10 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); pcu::Protect(); lion_set_verbosity(1); - if( !PCUObj.get()->Self() ) { + if( !PCUObj.Self() ) { lion_oprint(1,"PUMI Git hash %s\n", pumi_version()); lion_oprint(1,"PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); } @@ -56,8 +55,8 @@ int main(int argc, char** argv) std::string inputPath = "adapt.inp"; if(argc==2) inputPath = argv[1]; ph::Input in; - in.load(inputPath.c_str(), PCUObj.get()); - chef::cook(g,m,in,PCUObj.get()); + in.load(inputPath.c_str(), &PCUObj); + chef::cook(g,m,in,&PCUObj); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/phasta/chefStream.cc b/phasta/chefStream.cc index 3f4258184..37f79e8c4 100644 --- a/phasta/chefStream.cc +++ b/phasta/chefStream.cc @@ -18,7 +18,6 @@ #include #endif #include -#include namespace { void freeMesh(apf::Mesh* m) { @@ -35,7 +34,7 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); @@ -45,16 +44,16 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(PCUObj.get()); + GRStream* grs = makeGRStream(&PCUObj); ph::Input ctrl; - ctrl.load("adapt.inp", PCUObj.get()); - chef::cook(g,m,ctrl,grs,PCUObj.get()); - RStream* rs = makeRStream(PCUObj.get()); - attachRStream(grs,rs,PCUObj.get()); + ctrl.load("adapt.inp", &PCUObj); + chef::cook(g,m,ctrl,grs,&PCUObj); + RStream* rs = makeRStream(&PCUObj); + attachRStream(grs,rs,&PCUObj); reconfigureChef(ctrl); - chef::cook(g,m,ctrl,rs,PCUObj.get()); - destroyGRStream(grs,PCUObj.get()); - destroyRStream(rs,PCUObj.get()); + chef::cook(g,m,ctrl,rs,&PCUObj); + destroyGRStream(grs,&PCUObj); + destroyRStream(rs,&PCUObj); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); diff --git a/phasta/condense.cc b/phasta/condense.cc index 21e5e95d5..6e74a892b 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -11,7 +11,6 @@ #include #include #include -#include namespace { static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { @@ -42,10 +41,10 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); pcu::Protect(); lion_set_verbosity(1); - checkInputs(argc,argv,pcu_obj.get()); + checkInputs(argc,argv,&pcu_obj); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -54,9 +53,9 @@ int main(int argc, char** argv) { gmi_register_mesh(); GroupCode code; ph::Input in; - in.load(argv[1], pcu_obj.get()); + in.load(argv[1], &pcu_obj); in.openfile_read = openfile_read; - code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str(), pcu_obj.get()); + code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str(), &pcu_obj); chef::readAndAttachFields(in,code.mesh); apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); code.ctrl = in; diff --git a/phasta/cut_interface.cc b/phasta/cut_interface.cc index 151ae0885..bf4a9e131 100644 --- a/phasta/cut_interface.cc +++ b/phasta/cut_interface.cc @@ -14,7 +14,6 @@ #include #include #include -#include char const* modelfile; char const* attribfile; @@ -33,8 +32,8 @@ int main(int argc, char** argv) return 0; } { - auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - PCU_ALWAYS_ASSERT(PCUObj.get()->Peers() == 1); + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 1); #ifdef HAVE_SIMMETRIX SimModel_start(); Sim_readLicenseFile(0); @@ -61,7 +60,7 @@ int main(int argc, char** argv) gm = gmi_sim_load(modelfile, attribfile); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); - apf::Mesh2* m = ph::loadMesh(gm, meshfile, PCUObj.get()); + apf::Mesh2* m = ph::loadMesh(gm, meshfile, &PCUObj); m->verify(); #ifdef HAVE_SIMMETRIX if (ph::mesh_has_ext(meshfile, "sms")) diff --git a/phasta/migrate_interface.cc b/phasta/migrate_interface.cc index 5f907f238..e45bc3d67 100644 --- a/phasta/migrate_interface.cc +++ b/phasta/migrate_interface.cc @@ -16,7 +16,6 @@ #include #include #include -#include namespace { @@ -59,7 +58,7 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -69,11 +68,11 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv,pcu_obj.get()); + getConfig(argc,argv,&pcu_obj); gmi_model* gm; gm = gmi_sim_load(modelFile, attribFile); - apf::Mesh2* m = apf::loadMdsMesh(gm, inMesh, pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(gm, inMesh, &pcu_obj); m->verify(); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 2c5bd6590..880d5d7c9 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -19,7 +19,6 @@ #include #include #include -#include #include @@ -216,18 +215,18 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); MS_init(); SimModel_start(); Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv, pcu_obj.get()); + getConfig(argc, argv, &pcu_obj); if( should_log ) Sim_logOn("convert.sim.log"); if (should_attach_order && should_fix_pyramids) { - if (!pcu_obj.get()->Self()) + if (!pcu_obj.Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = false; } @@ -246,9 +245,9 @@ int main(int argc, char** argv) double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); double t1 = pcu::Time(); - if(!pcu_obj.get()->Self()) + if(!pcu_obj.Self()) lion_eprint(1, "read and created the simmetrix mesh in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, pcu_obj.get()); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, &pcu_obj); double t2 = pcu::Time(); if(!simApfMesh->getPCU()->Self()) lion_eprint(1, "created the apf_sim mesh in %f seconds\n", t2-t1); diff --git a/phasta/readUrPrep.cc b/phasta/readUrPrep.cc index d6ea4e1e5..71eba7065 100644 --- a/phasta/readUrPrep.cc +++ b/phasta/readUrPrep.cc @@ -4,7 +4,6 @@ #include #include #include -#include namespace { void freeMesh(apf::Mesh* m) { @@ -20,12 +19,12 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); pcu::Protect(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], &pcu_obj); ph::Input ctrl; - ctrl.load(argv[3], pcu_obj.get()); + ctrl.load(argv[3], &pcu_obj); ctrl.openfile_read = openfile_read; ctrl.buildMapping = 0; //can't map new vertices from UR chef::readAndAttachFields(ctrl,m); diff --git a/phasta/threshold.cc b/phasta/threshold.cc index f8e807d80..b619bf0a9 100644 --- a/phasta/threshold.cc +++ b/phasta/threshold.cc @@ -4,7 +4,6 @@ #include #include #include -#include #ifdef HAVE_SIMMETRIX #include #include @@ -21,7 +20,7 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); { - auto pcu_obj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -30,7 +29,7 @@ int main(int argc, char** argv) gmi_register_mesh(); lion_set_verbosity(1); ph::Input in; - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], pcu_obj.get()); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); m->verify(); in.restartFileName = argv[3]; in.timeStepNumber = 0; From 807746cf4cb9ada8240f2ce52c5c57f6cc7a18c1 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 29 Aug 2024 10:51:39 -0400 Subject: [PATCH 139/141] Removed semi colon after Macro --- pcu/PCU.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcu/PCU.h b/pcu/PCU.h index 40dc3ed39..3edf052fc 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -79,7 +79,7 @@ class PCU { int Received(size_t *size) noexcept; void *Extract(size_t size) noexcept; - void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3); + void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3) void DebugPrint(const char* format, va_list args) noexcept; /* Debug functions */ void DebugOpen() noexcept; From 7e4cd5a1b6083248712496593eeb4803ae147af8 Mon Sep 17 00:00:00 2001 From: flagdanger Date: Mon, 23 Sep 2024 19:44:16 -0400 Subject: [PATCH 140/141] CGNS fixes --- apf/apfCGNS.cc | 6 +++--- mds/mdsCGNS.cc | 10 +++++----- test/convert.cc | 9 +++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 92eb98782..4bd3499b0 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -692,7 +692,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, return std::make_pair(cacheStarts, cacheEnds); }; - const auto globalElementList = [](const std::vector &bcList, std::vector &allElements) { + const auto globalElementList = [&m](const std::vector &bcList, std::vector &allElements) { std::vector sizes(m->getPCU()->Peers(), 0); // important initialiser const int l = bcList.size(); MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1, @@ -1049,11 +1049,11 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap auto communicator = m->getPCU()->GetMPIComm(); cgp_mpi_comm(communicator); // - cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); + cgp_pio_mode(CGP_INDEPENDENT); CGNS cgns; cgns.fname = std::string(prefix); - if (cgp_open(prefix, CGNS_ENUMV(CG_MODE_WRITE), &cgns.index)) + if (cgp_open(prefix, CG_MODE_WRITE, &cgns.index)) cgp_error_exit(); { diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index 24ad15f95..57e32ca27 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -177,19 +177,19 @@ struct MeshDataGroup if (components.size() == 1) { std::cout << "Scalar Group has " << components.size() << " related componenets: " << std::endl; - for (const auto m : components) + for (const auto& m : components) std::cout << "Field " << m.second.name << " @ " << m.second.si << " " << m.second.fi << std::endl; } else if (components.size() == 3) { std::cout << "Vector Group has " << components.size() << " related componenets: " << std::endl; - for (const auto m : components) + for (const auto& m : components) std::cout << "Field " << m.second.name << " @ " << m.second.si << " " << m.second.fi << std::endl; } else if (components.size() == 9) { std::cout << "Matrix Group has " << components.size() << " related componenets: " << std::endl; - for (const auto m : components) + for (const auto& m : components) std::cout << "Field " << m.second.name << " @ " << m.second.si << " " << m.second.fi << std::endl; } else @@ -1056,8 +1056,8 @@ apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap int cgid = -1; auto comm = PCU_Get_Comm(h); cgp_mpi_comm(comm); - cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); - cgp_open(fname.c_str(), CGNS_ENUMV(CG_MODE_READ), &cgid); + cgp_pio_mode(CGP_INDEPENDENT); + cgp_open(fname.c_str(), CG_MODE_READ, &cgid); int nbases = -1; cg_nbases(cgid, &nbases); diff --git a/test/convert.cc b/test/convert.cc index b1cf34bcd..12e965c5a 100644 --- a/test/convert.cc +++ b/test/convert.cc @@ -22,6 +22,7 @@ #include #include #include +#include using namespace std; @@ -245,7 +246,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int nvert=i; PList_delete(listV); - double coordNewPt[nvert][3]; + std::vector> coordNewPt(nvert,{0,0,0}); for(i=0; i< nvert ; i++) { int* markedData; if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){ // not sure about marked yet @@ -346,13 +347,13 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int* vtxData = new int[1]; vtxData[0] = count2D; EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData); - V_coord(vrts[i],coordNewPt[i]); + V_coord(vrts[i],coordNewPt[i].data()); fprintf ( fcr, "%.15E %.15E %d %d %d %d \n", coordNewPt[i][0],coordNewPt[i][1], vClassDim, foundESTag, foundETag, foundEETag ); } } - double coordFather[nvert][3]; + std::vector> coordFather(nvert,{0,0,0}); int fatherIds[4]; //store the ids of the fathers (vertices) on the root face for(i=0; i< nvert ; i++) { int* fatherIdPtr; @@ -364,7 +365,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c } assert(exists); fatherIds[i] = fatherIdPtr[0]; - V_coord(vrts[i],coordFather[i]); + V_coord(vrts[i],coordFather[i].data()); fprintf ( fcn, "%d ", fatherIds[i]); } fprintf ( fcn, "\n"); From 6e47c85c33db0da7aaa0e10e5474834326f4539b Mon Sep 17 00:00:00 2001 From: flagdanger Date: Thu, 10 Oct 2024 19:27:08 -0400 Subject: [PATCH 141/141] cgns fixes --- apf/apfMesh.h | 2 +- mds/apfMDS.h | 10 ++++------ mds/mdsCGNS.cc | 18 +++--------------- pcu/pkg_tribits.cmake | 1 + test/cgns.cc | 2 +- 5 files changed, 10 insertions(+), 23 deletions(-) diff --git a/apf/apfMesh.h b/apf/apfMesh.h index c7b92b812..d862333f2 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -396,7 +396,7 @@ class Mesh GlobalNumbering* findGlobalNumbering(const char* name); GlobalNumbering* getGlobalNumbering(int i); - /** \brief get the global pcu */ + /** \brief get the mesh pcu */ pcu::PCU* getPCU() const {return pcu_;} void switchPCU(pcu::PCU *newPCU); /** \brief true if any associated fields use array storage */ diff --git a/mds/apfMDS.h b/mds/apfMDS.h index a44a2f74d..47d4cfbaf 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -36,6 +36,8 @@ // AJP: alternative is to allow common cgns base header // but trying to avoid that since it's not core functionality #include +#include + // namespace pcu{ class PCU; @@ -54,8 +56,6 @@ class Field; /** \brief a map from global ids to vertex objects */ typedef std::map GlobalToVert; -typedef struct PCU_t PCU_t; - /** \brief create an empty MDS part \param model the geometric model interface @@ -204,12 +204,10 @@ int getMdsIndex(Mesh2* in, MeshEntity* e); so call apf::reorderMdsMesh after any mesh modification. */ MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index); -Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); -Mesh2* loadMdsFromCGNS2(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); +Mesh2* loadMdsFromCGNS(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); // names of mesh data to read from file: (VERTEX, VelocityX; CellCentre, Pressure) -Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); -Mesh2* loadMdsFromCGNS2(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); +Mesh2* loadMdsFromCGNS(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); int gmshMajorVersion(const char* filename); diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index 57e32ca27..fd811f914 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -231,7 +231,7 @@ void Kill(PCU_t h, const int fid, Args &&... args) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free(h); + PCU_Comm_Free(&h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -252,7 +252,7 @@ void Kill(PCU_t h, const int fid) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free(h); + PCU_Comm_Free(&h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -1286,7 +1286,7 @@ apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap }; // process meshData and find vectors (and matrices) - const auto findMatch = [&meshData, &index, &cgid](MeshData &other, std::vector &meshDataGroups) { + const auto findMatch = [&meshData, &index, &cgid, &h](MeshData &other, std::vector &meshDataGroups) { MeshDataGroup group; for (auto &md : meshData) { @@ -1723,12 +1723,6 @@ namespace apf { // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. -Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) -{ - PCU_t h = PCU_Get_Global_Handle(); - return loadMdsFromCGNS(h, g, fname, cgnsBCMap, meshData); -} - Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) { #ifdef HAVE_CGNS @@ -1744,12 +1738,6 @@ Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap } // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. -Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) -{ - PCU_t h = PCU_Get_Global_Handle(); - return loadMdsFromCGNS(h, g, fname, cgnsBCMap); -} - Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) { #ifdef HAVE_CGNS diff --git a/pcu/pkg_tribits.cmake b/pcu/pkg_tribits.cmake index ec9a43a16..86278a4a2 100644 --- a/pcu/pkg_tribits.cmake +++ b/pcu/pkg_tribits.cmake @@ -54,6 +54,7 @@ set(HEADERS pcu_util.h reel/reel.h pcu_defines.h + PCU.h ) tribits_add_library( pcu diff --git a/test/cgns.cc b/test/cgns.cc index dbce1cc10..d84dc96c5 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -179,7 +179,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: gmi_model *g = gmi_load(".null"); PCU_t h; h.ptr = static_cast(PCUObj); - apf::Mesh2 *m = apf::loadMdsFromCGNS2(h, g, argv1.c_str(), cgnsBCMap, meshData); + apf::Mesh2 *m = apf::loadMdsFromCGNS(h, g, argv1.c_str(), cgnsBCMap, meshData); m->verify(); // m->writeNative(argv2.c_str());